diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
index f13502bf..47f91ede 100644
--- a/.github/workflows/macos.yml
+++ b/.github/workflows/macos.yml
@@ -22,7 +22,10 @@ jobs:
xcode-version: latest-stable
- name: Install dependencies
- run: brew install ninja
+ run: |
+ brew update && brew upgrade && brew upgrade --cask && brew cleanup
+ brew install ninja
+ echo "CC=$(echo ${{matrix.compiler}} | sed -e 's/^g++/gcc/' | sed 's/+//g')" >> $GITHUB_ENV
- name: Configure
run: cmake -DCCT_ENABLE_ASAN=OFF -S . -B build -GNinja
diff --git a/.github/workflows/ubuntu-clang-tidy.yml b/.github/workflows/ubuntu-clang-tidy.yml
index 240681e4..f2d2fbf5 100644
--- a/.github/workflows/ubuntu-clang-tidy.yml
+++ b/.github/workflows/ubuntu-clang-tidy.yml
@@ -37,6 +37,7 @@ jobs:
clang-tidy --dump-config
cmake -S . -B build -DCCT_ENABLE_CLANG_TIDY=ON -DCCT_ENABLE_ASAN=OFF -GNinja
env:
+ CC: clang-${{matrix.clang-version}}
CXX: clang++-${{matrix.clang-version}}
CMAKE_BUILD_TYPE: ${{matrix.buildmode}}
diff --git a/.github/workflows/ubuntu-monitoring.yml b/.github/workflows/ubuntu-special.yml
similarity index 71%
rename from .github/workflows/ubuntu-monitoring.yml
rename to .github/workflows/ubuntu-special.yml
index d91c0c42..de09863a 100644
--- a/.github/workflows/ubuntu-monitoring.yml
+++ b/.github/workflows/ubuntu-special.yml
@@ -1,4 +1,4 @@
-name: Monitoring
+name: Special
on:
push:
@@ -7,14 +7,15 @@ on:
pull_request:
jobs:
- ubuntu-monitoring-build:
- name: Build on Ubuntu with monitoring support
+ ubuntu-special-build:
+ name: Build on Ubuntu with monitoring / protobuf support
runs-on: ubuntu-latest
strategy:
matrix:
compiler: [g++-11]
buildmode: [Debug]
- build-prometheus-from-source: [0, 1]
+ build-special-from-source: [0, 1]
+ prometheus-options: ["-DBUILD_SHARED_LIBS=ON -DENABLE_PULL=OFF -DENABLE_PUSH=ON -DENABLE_COMPRESSION=OFF -DENABLE_TESTING=OFF"]
steps:
- name: Checkout repository code
@@ -24,6 +25,7 @@ jobs:
run: |
sudo apt update
sudo apt install cmake libssl-dev git libcurl4-openssl-dev ninja-build -y --no-install-recommends
+ echo "CC=$(echo ${{matrix.compiler}} | sed -e 's/^g++/gcc/' | sed 's/+//g')" >> $GITHUB_ENV
- name: Install prometheus-cpp
run: |
@@ -39,10 +41,10 @@ jobs:
env:
CXX: ${{matrix.compiler}}
CMAKE_BUILD_TYPE: ${{matrix.buildmode}}
- if: matrix.build-prometheus-from-source == 0
+ if: matrix.build-special-from-source == 0
- name: Configure CMake
- run: cmake -S . -B build -DCCT_BUILD_PROMETHEUS_FROM_SRC=${{matrix.build-prometheus-from-source}} -DCCT_ENABLE_ASAN=OFF -GNinja
+ run: cmake -S . -B build -DCCT_BUILD_PROMETHEUS_FROM_SRC=${{matrix.build-special-from-source}} -DCCT_ENABLE_PROTO=${{matrix.build-special-from-source}} -DCCT_ENABLE_ASAN=OFF -GNinja
env:
CXX: ${{matrix.compiler}}
CMAKE_BUILD_TYPE: ${{matrix.buildmode}}
@@ -56,4 +58,4 @@ jobs:
- name: Sanity check main executable
working-directory: ${{github.workspace}}/build
- run: ./coincenter --help
\ No newline at end of file
+ run: ./coincenter --help
diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml
index a272d084..4847b203 100644
--- a/.github/workflows/ubuntu.yml
+++ b/.github/workflows/ubuntu.yml
@@ -23,6 +23,7 @@ jobs:
run: |
sudo apt update
sudo apt install cmake libssl-dev libcurl4-openssl-dev ninja-build -y --no-install-recommends
+ echo "CC=$(echo ${{matrix.compiler}} | sed -e 's/^g++/gcc/' | sed 's/+//g')" >> $GITHUB_ENV
- name: Install gcc
run: |
@@ -30,7 +31,7 @@ jobs:
# Temporary workaround for libasan bug stated here: https://github.com/google/sanitizers/issues/1716
sudo sysctl vm.mmap_rnd_bits=28
- if: startsWith(matrix.compiler, 'g++')
+ if: startsWith(matrix.compiler, 'g')
- name: Install clang
run: |
@@ -38,7 +39,7 @@ jobs:
wget https://apt.llvm.org/llvm.sh
chmod +x llvm.sh
sudo ./llvm.sh ${CLANG_VERSION}
- if: startsWith(matrix.compiler, 'clang++')
+ if: startsWith(matrix.compiler, 'clang')
- name: Configure CMake
run: cmake -S . -B build -GNinja
@@ -55,4 +56,4 @@ jobs:
- name: Sanity check main executable
working-directory: ${{github.workspace}}/build
- run: ./coincenter --help
\ No newline at end of file
+ run: ./coincenter --help
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
index f579676c..e733879d 100644
--- a/.github/workflows/windows.yml
+++ b/.github/workflows/windows.yml
@@ -33,4 +33,4 @@ jobs:
- name: Sanity check main executable
working-directory: ${{github.workspace}}/build/${{matrix.buildmode}}
- run: .\coincenter.exe --help
\ No newline at end of file
+ run: .\coincenter.exe --help
diff --git a/.gitignore b/.gitignore
index 9ad9a59a..44d7947c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -64,6 +64,7 @@ data/cache
data/log
data/secret
!data/secret/secret_test.json
+data/serialized
data/static/exchangeconfig.json
data/static/generalconfig.json
monitoring/data/grafana/*
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4d6f5bbe..a6fd3393 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -38,6 +38,7 @@ option(CCT_ENABLE_TESTS "Build the unit tests" ${MAIN_PROJECT})
option(CCT_BUILD_EXEC "Build an executable instead of a static library" ${MAIN_PROJECT})
option(CCT_ENABLE_ASAN "Compile with AddressSanitizer" ${CCT_ASAN_BUILD})
option(CCT_ENABLE_CLANG_TIDY "Compile with clang-tidy checks" OFF)
+option(CCT_ENABLE_PROTO "Compile with protobuf support (to export data to the outside world)" ON)
option(CCT_BUILD_PROMETHEUS_FROM_SRC "Fetch and build from prometheus-cpp sources" OFF)
set(CCT_DATA_DIR "${CMAKE_CURRENT_SOURCE_DIR}/data" CACHE PATH "Needed data directory for coincenter. Can also be overridden at runtime with this environment variable")
@@ -162,14 +163,68 @@ set(JWT_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
list(APPEND fetchContentPackagesToMakeAvailable jwt-cpp)
+# protobuf - serialization / deserialization library
+set(PROTOBUF_FETCHED_CONTENT OFF)
+if(CCT_ENABLE_PROTO)
+ find_package(Protobuf CONFIG)
+ if(Protobuf_FOUND)
+ message(STATUS "Linking with protobuf ${protobuf_VERSION}")
+ else()
+ # Check here for a new version: https://protobuf.dev/support/version-support/#cpp
+ if (NOT PROTOBUF_VERSION)
+ set(PROTOBUF_VERSION v5.26.1)
+ endif()
+
+ message(STATUS "Configuring protobuf ${PROTOBUF_VERSION} from sources")
+
+ # Using git here to simplify cmake code instead of archive download as
+ # it depends on Abseil behind the scene whose code is not included in the release archives.
+ FetchContent_Declare(
+ protobuf
+ GIT_REPOSITORY https://github.com/protocolbuffers/protobuf.git
+ GIT_TAG ${PROTOBUF_VERSION}
+ GIT_SHALLOW true
+ )
+
+ set(protobuf_BUILD_TESTS OFF CACHE BOOL "" FORCE)
+ set(protobuf_BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)
+ set(protobuf_BUILD_LIBUPB OFF CACHE BOOL "" FORCE)
+
+ # Abseil options
+ set(ABSL_CXX_STANDARD ${CMAKE_CXX_STANDARD} CACHE STRING "" FORCE)
+ set(ABSL_PROPAGATE_CXX_STD ON CACHE BOOL "" FORCE)
+ set(ABSL_ENABLE_INSTALL OFF CACHE BOOL "" FORCE)
+ set(ABSL_BUILD_TESTING OFF CACHE BOOL "" FORCE)
+
+ if(CCACHE_PROGRAM)
+ set(protobuf_ALLOW_CCACHE ON CACHE BOOL "" FORCE)
+ endif()
+
+ set(protobuf_INSTALL OFF CACHE BOOL "" FORCE)
+ set(protobuf_BUILD_LIBUPB OFF CACHE BOOL "" FORCE)
+ set(protobuf_WITH_ZLIB ON CACHE BOOL "" FORCE)
+ SET(protobuf_MSVC_STATIC_RUNTIME OFF CACHE INTERNAL "")
+
+
+ list(APPEND fetchContentPackagesToMakeAvailable protobuf)
+
+ set(PROTOBUF_FETCHED_CONTENT ON)
+ endif()
+endif()
+
# Make fetch content available
if(fetchContentPackagesToMakeAvailable)
message(STATUS "Configuring packages ${fetchContentPackagesToMakeAvailable}")
+
FetchContent_MakeAvailable("${fetchContentPackagesToMakeAvailable}")
+
+ if(PROTOBUF_FETCHED_CONTENT)
+ include(${protobuf_SOURCE_DIR}/cmake/protobuf-generate.cmake)
+ endif()
endif()
# Unit Tests
-include(cmake/AddUnitTest.cmake)
+include(${CMAKE_CURRENT_LIST_DIR}/cmake/AddUnitTest.cmake)
# Compiler warnings (only in Debug mode)
if(CMAKE_BUILD_TYPE STREQUAL "Debug")
@@ -215,13 +270,20 @@ if(CCT_ENABLE_PROMETHEUS)
add_compile_definitions(CCT_ENABLE_PROMETHEUS)
endif()
+if(CCT_ENABLE_PROTO)
+ add_compile_definitions(CCT_ENABLE_PROTO)
+ add_compile_definitions("CCT_PROTOBUF_VERSION=\"${PROTOBUF_VERSION}\"")
+endif()
+
# Link to sub folders CMakeLists.txt, from the lowest level to the highest level for documentation
# (beware of cyclic dependencies)
add_subdirectory(src/tech)
add_subdirectory(src/monitoring)
add_subdirectory(src/http-request)
add_subdirectory(src/objects)
+add_subdirectory(src/serialization)
add_subdirectory(src/api-objects)
+add_subdirectory(src/trading)
add_subdirectory(src/api)
add_subdirectory(src/engine)
add_subdirectory(src/main)
diff --git a/CONFIG.md b/CONFIG.md
index 9b630a7a..b4222176 100644
--- a/CONFIG.md
+++ b/CONFIG.md
@@ -48,18 +48,19 @@ Configures the logging, tracking activity of relevant commands, and console outp
#### General options description
-| Name | Value | Description |
-| ---------------------------------------------- | ---------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| **apiOutputType** | String among {`off`, `table`, `json`} | Configure the default output type of coincenter (can be overridden by command line)queries |
-| **fiatConversion.rate** | Duration string (ex: `8h`) | Minimum duration between two consecutive requests of the same fiat conversion |
-| **log.activityTracking.commandTypes** | Array of strings (ex: `["Buy", "Sell"]`) | Array of command types whose output will be stored to activity history files. |
-| **log.activityTracking.dateFileNameFormat** | String (ex: `%Y-%m` for month split) | Defines the date string format suffix used by activity history files. The string should be compatible with [std::strftime](https://en.cppreference.com/w/cpp/chrono/c/strftime). Old data will never be clean-up by `coincenter` (as it may contain important data). User should manage the clean-up / storage. |
-| **log.activityTracking.withSimulatedCommands** | Boolean | When some commands are launched in simulated mode (trades, withdraw for instance), they will be logged if `true`. |
-| **log.consoleLevel** | String | Defines the log level for standard output. Can be {'off', 'critical', 'error', 'warning', 'info', 'debug', 'trace'} |
-| **log.fileLevel** | String | Defines the log level in files. Can be {'off', 'critical', 'error', 'warning', 'info', 'debug', 'trace'} |
-| **log.maxFileSize** | String (ex: `5Mi` for 5 Megabytes) | Defines in bytes the maximum logging file size. A string representation of an integral, possibly with one suffix ending such as k, M, G, T (1k multipliers) or Ki, Mi, Gi, Ti (1024 multipliers) are supported. |
-| **log.maxNbFiles** | Integer | Number of maximum rotating files for log in files |
-| **requests.concurrency.nbMaxParallelRequests** | Integer | Size of the thread pool that makes exchange requests. |
+| Name | Value | Description |
+| -------------------------------------------------------- | ---------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| **apiOutputType** | String among {`off`, `table`, `json`} | Configure the default output type of coincenter (can be overridden by command line)queries |
+| **fiatConversion.rate** | Duration string (ex: `8h`) | Minimum duration between two consecutive requests of the same fiat conversion |
+| **log.activityTracking.commandTypes** | Array of strings (ex: `["Buy", "Sell"]`) | Array of command types whose output will be stored to activity history files. |
+| **log.activityTracking.dateFileNameFormat** | String (ex: `%Y-%m` for month split) | Defines the date string format suffix used by activity history files. The string should be compatible with [std::strftime](https://en.cppreference.com/w/cpp/chrono/c/strftime). Old data will never be clean-up by `coincenter` (as it may contain important data). User should manage the clean-up / storage. |
+| **log.activityTracking.withSimulatedCommands** | Boolean | When some commands are launched in simulated mode (trades, withdraw for instance), they will be logged if `true`. |
+| **log.consoleLevel** | String | Defines the log level for standard output. Can be {'off', 'critical', 'error', 'warning', 'info', 'debug', 'trace'} |
+| **log.fileLevel** | String | Defines the log level in files. Can be {'off', 'critical', 'error', 'warning', 'info', 'debug', 'trace'} |
+| **log.maxFileSize** | String (ex: `5Mi` for 5 Megabytes) | Defines in bytes the maximum logging file size. A string representation of an integral, possibly with one suffix ending such as k, M, G, T (1k multipliers) or Ki, Mi, Gi, Ti (1024 multipliers) are supported. |
+| **log.maxNbFiles** | Integer | Number of maximum rotating files for log in files |
+| **requests.concurrency.nbMaxParallelRequests** | Integer | Size of the thread pool that makes exchange requests. |
+| **trading.automation.deserialization.loadChunkDuration** | Duration string (ex: `1w`) | Time window duration of historic stored data loaded and replayed at once given to the trading engine |
### static/exchangeconfig.json
@@ -152,6 +153,7 @@ Refer to the hardcoded default json example as a model in case of doubt.
| *query* | **updateFrequency.depositWallet** | Duration string (ex: `1min`) | Minimum duration between two consecutive requests of deposit information (including wallet) |
| *query* | **updateFrequency.currencyInfo** | Duration string (ex: `4h`) | Minimum duration between two consecutive requests of dynamic currency info retrieval on Bithumb only (used for place order) |
| *query* | **placeSimulateRealOrder** | Boolean (`true` or `false`) | If `true`, in trade simulation mode (with `--sim`) exchanges which do not support simulated mode in place order will actually place a real order, with the following characteristics:
- trade strategy forced to `maker`
- price will be changed to a maximum for a sell, to a minimum for a buy
This will allow place of a 'real' order that cannot be matched in practice (if it is, lucky you!) |
+| *query* | **marketDataSerialization** | Boolean (`true` or `false`) | If `true` and `coincenter` is compiled with **protobuf** support, some market data will automatically be exported in the `data/serialization` directory (`orderbook` and `last-trades`) for a long term storage |
| *query* | **multiTradeAllowedByDefault** | Boolean (`true` or `false`) | If `true`, [multi-trade](README.md#multi-trade) will be allowed by default for `trade`, `buy` and `sell`. It can be overridden at command line level with `--no-multi-trade` and `--multi-trade`. |
| *query* | **validateApiKey** | Boolean (`true` or `false`) | If `true`, each loaded private key will be tested at start of the program. In case of a failure, it will be removed from the list of private accounts loaded by `coincenter`, so that later queries do not consider it instead of raising a runtime exception. The downside is that it will make an additional check that will make startup slower. | |
| *tradefees* | **maker** | String as decimal number representing a percentage (for instance, "0.15") | Trade fees occurring when a maker order is matched |
diff --git a/Dockerfile b/Dockerfile
index ec800a9f..bb189794 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -5,11 +5,12 @@ ARG BUILD_MODE=Release
ARG BUILD_TEST=0
ARG BUILD_ASAN=0
ARG BUILD_WITH_PROMETHEUS=1
+ARG BUILD_WITH_PROTOBUF=1
# Install base & build dependencies, needed certificates for curl to work with https
RUN apt update && \
apt upgrade -y && \
- apt install build-essential ninja-build libssl-dev zlib1g-dev libcurl4-openssl-dev cmake ca-certificates -y --no-install-recommends
+ apt install build-essential ninja-build libssl-dev zlib1g-dev libcurl4-openssl-dev cmake git ca-certificates -y --no-install-recommends
# Copy source files
WORKDIR /app/src
@@ -41,6 +42,7 @@ RUN cmake -DCMAKE_BUILD_TYPE=${BUILD_MODE} \
-DCCT_ENABLE_TESTS=${BUILD_TEST} \
-DCCT_ENABLE_ASAN=${BUILD_ASAN} \
-DCCT_BUILD_PROMETHEUS_FROM_SRC=${BUILD_WITH_PROMETHEUS} \
+ -DCCT_ENABLE_PROTO=${BUILD_WITH_PROTOBUF} \
-GNinja ..
# Build
diff --git a/INSTALL.md b/INSTALL.md
index db8df724..dfe33fc2 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -158,6 +158,7 @@ The minimum tested version is cmake `3.15`, but it's recommended that you use th
| `CCT_BUILD_EXEC` | `ON` if main project | Build an executable instead of a static library |
| `CCT_ENABLE_ASAN` | `ON` if Debug mode | Compile with AddressSanitizer |
| `CCT_ENABLE_CLANG_TIDY` | `ON` if Debug mode and `clang-tidy` is found in `PATH` | Compile with clang-tidy checks |
+| `CCT_ENABLE_PROTO` | `ON` | Compile with protobuf support |
Example on Linux: to compile it in `Release` mode and `ninja` generator
diff --git a/README.md b/README.md
index 0169bbff..110058b7 100644
--- a/README.md
+++ b/README.md
@@ -38,7 +38,16 @@ Main features:
- Cancel opened orders
- Withdraw (with check at destination that funds are well received)
- Dust sweeper
-
+
+## Market data storage & replay
+
+`coincenter` is able to store the following market data in serialized [binary protobuf](https://protobuf.dev/) format for offline trading algorithm replay:
+
+- Market order book
+- Public trades
+
+Refer to the dedicated [documentation page](TRADING.md) for more information.
+
## Supported exchanges
| Exchange | Link |
@@ -56,6 +65,7 @@ Main features:
- [coincenter](#coincenter)
- [Market Data](#market-data)
- [Account requests](#account-requests)
+ - [Market data storage \& replay](#market-data-storage--replay)
- [Supported exchanges](#supported-exchanges)
- [About](#about)
- [Installation](#installation)
@@ -1298,4 +1308,4 @@ Possible output:
| kucoin | 6090943.32410022531 SHIB |
| upbit | 6084383.631243834 SHIB |
+----------+------------------------------+
-```
\ No newline at end of file
+```
diff --git a/TRADING.md b/TRADING.md
new file mode 100644
index 00000000..0e45c519
--- /dev/null
+++ b/TRADING.md
@@ -0,0 +1,110 @@
+# Trading
+
+`coincenter` is able to serialize market data and deserialize it later on for future usage.
+
+## Overview
+
+Currently, these two sources of data are serializable into [protobuf](https://protobuf.dev/) objects:
+
+- Market order books
+- Public trades
+
+## Configuration
+
+### Compilation
+
+By default, `coincenter` will be built with **protobuf** support (controlled by `cmake` flag `CCT_ENABLE_PROTO` that defaults to `ON`).
+
+It will try to link to a known installation of **protobuf** if found on current system (oldest tested version is `v25`, make sure to use this version at least), otherwise it will download and compile it from sources.
+
+### Serialization configuration
+
+To be able to serialize market data on disk, make sure that you set the **marketDataSerialization** variable to `true` in `exchangeconfig.json` for the exchanges you would like to interact with.
+See the [exchange configuration part](CONFIG.md#exchanges-options-description) for more information about how to configure it.
+
+## Serialization of market data
+
+The data will be organized by exchange, then market (asset pair, for instance `BTC-USD`), and finally dates (with directories from **year**, **month**, and finally **day** and files as **hours**).
+
+All will be stored in `coincenter` data directory, under `serialized` sub folder.
+
+Here is an example of the structure of files you will obtain:
+
+```bash
+data/serialized//
+├── binance
+│ ├── BTC-EUR
+│ │ └── 2024
+│ │ └── 01
+│ │ ├── 11
+│ │ │ └── 22-00-00_22-59-59.binpb
+│ │ ├── 12
+│ │ │ ├── 08-00-00_08-59-59.binpb
+│ │ │ ├── 09-00-00_09-59-59.binpb
+│ │ │ └── 11-00-00_11-59-59.binpb
+│ │ └── 14
+│ │ ├── 07-00-00_07-59-59.binpb
+│ │ └── 08-00-00_08-59-59.binpb
+│ ├── ETH-USDT
+│ │ └── 2024
+│ │ ├── 01
+│ │ │ ├── 09
+│ │ │ │ ├── 08-00-00_08-59-59.binpb
+│ │ │ │ ├── 09-00-00_09-59-59.binpb
+│ │ │ │ ├── 10-00-00_10-59-59.binpb
+│ │ │ │ ├── 11-00-00_11-59-59.binpb
+├── huobi
+│ ├── ADA-USDT
+│ │ └── 2024
+│ │ └── 02
+│ │ └── 10
+│ │ └── 16-00-00_16-59-59.binpb
+│ ├── BTC-EUR
+│ │ └── 2024
+│ │ └── 01
+│ │ ├── 11
+│ │ │ └── 22-00-00_22-59-59.binpb
+....
+```
+
+To retrieve market data, it's possible to either use multi-commands with both `orderbook` and `last-trades` commands stacked together, or you can use the more handy `market-data` option that is basically a combination of the two without the output by default (it has been created only for serialization purposes).
+
+For instance, to retrieve continuously data and serialize them indefinitely, you can use the following command:
+
+```bash
+coincenter -r --repeat-time 2s --log warning \
+ market-data btc-eur,binance \
+ market-data eth-usdt,kucoin \
+ market-data ada-usdt,huobi \
+ market-data btc-eur,kraken
+```
+
+Note the usage of the `-r` (repeat option) to keep querying the data as long as you leave `coincenter` up. It's a good idea to also limit the number of logs with setting console log level to `warning` but not mandatory.
+
+With this command running for an extended period of time, you should obtain a list of files like in the above example.
+
+Stacking `market-data` commands together with different exchanges (like in the above example) will allow `coincenter` to perform the queries in parallel, ensuring optimal frequency of data updates. This optimization may be implemented for other commands in the future, but it's currently supported only for `market-data`.
+
+### Graceful shutdown
+
+Data is flushed on the disk at regular intervals (around 10 minutes). If you wish to restart / shutdown `coincenter` with an infinite `repeat` command to store continuously market data, you can send `SIGINT` or `SIGTERM` so that `coincenter` can gracefully stop after current request and flush its remaining data on disk before shutdown.
+
+## Replaying historic market data
+
+Of course, serialization is useful only if we re-use the data one day. Being **protobuf**, not only `coincenter` could read them, but also other tools, but `coincenter` is also able to read this data.
+
+Locate the `.proto` files in order and feed them to the external tool in case you want to use another program to read the data. Data is stored in hours in streaming mode (see documentation [here](https://protobuf.dev/programming-guides/techniques/#streaming)), and written in compressed format.
+
+If you want to use a third-party tool to read this data, locate the `.proto` files in the `src` directory for your external program to be able to deserialize the `.binpb` files.
+
+We will focus here on `coincenter` features concerning this data.
+
+### Experimental - Testing trading algorithms
+
+`coincenter` embeds a trading simulator engine that is able to be used for any custom trading algorithm that would derive from the interface.
+
+This trading simulator will read chunks of historic data stored in **protobuf** and inject them in trading algorithms.
+
+Locate the `AbstractMarketTrader` class and derive it - you need to return a `TraderCommand` for each market order book and a list of last public trades that occurred at this specific point of time.
+
+TODO: extend this documentation.
diff --git a/alpine.Dockerfile b/alpine.Dockerfile
index bd083fab..cbca0fb8 100644
--- a/alpine.Dockerfile
+++ b/alpine.Dockerfile
@@ -5,9 +5,10 @@ ARG BUILD_MODE=Release
ARG BUILD_TEST=0
ARG BUILD_ASAN=0
ARG BUILD_WITH_PROMETHEUS=1
+ARG BUILD_WITH_PROTOBUF=1
# Install base & build dependencies, needed certificates for curl to work with https
-RUN apk add --update --upgrade --no-cache g++ libc-dev openssl-dev zlib-dev curl-dev cmake ninja ca-certificates
+RUN apk add --update --upgrade --no-cache linux-headers g++ zlib-dev openssl-dev curl-dev cmake ninja git ca-certificates
# Copy source files
WORKDIR /app/src
@@ -39,6 +40,7 @@ RUN cmake -DCMAKE_BUILD_TYPE=${BUILD_MODE} \
-DCCT_ENABLE_TESTS=${BUILD_TEST} \
-DCCT_ENABLE_ASAN=${BUILD_ASAN} \
-DCCT_BUILD_PROMETHEUS_FROM_SRC=${BUILD_WITH_PROMETHEUS} \
+ -DCCT_ENABLE_PROTO=${BUILD_WITH_PROTOBUF} \
-GNinja ..
# Build
diff --git a/src/api/common/CMakeLists.txt b/src/api/common/CMakeLists.txt
index 9464b950..76480e03 100644
--- a/src/api/common/CMakeLists.txt
+++ b/src/api/common/CMakeLists.txt
@@ -1,12 +1,12 @@
aux_source_directory(src API_COMMON_SRC)
-include(FindOpenSSL)
-
add_library(coincenter_api-common STATIC ${API_COMMON_SRC})
+
target_link_libraries(coincenter_api-common PUBLIC coincenter_api-objects)
target_link_libraries(coincenter_api-common PUBLIC coincenter_objects)
target_link_libraries(coincenter_api-common PUBLIC coincenter_http-request)
+target_link_libraries(coincenter_api-common PUBLIC coincenter_serialization)
target_link_libraries(coincenter_api-common PRIVATE OpenSSL::SSL)
target_include_directories(coincenter_api-common PUBLIC include)
diff --git a/src/api/common/include/exchangepublicapi.hpp b/src/api/common/include/exchangepublicapi.hpp
index f0e24a5b..586e76ca 100644
--- a/src/api/common/include/exchangepublicapi.hpp
+++ b/src/api/common/include/exchangepublicapi.hpp
@@ -1,5 +1,6 @@
#pragma once
+#include
#include
#include
#include
@@ -10,15 +11,20 @@
#include "currencyexchangeflatset.hpp"
#include "exchangebase.hpp"
#include "exchangepublicapitypes.hpp"
+#include "market-order-book-vector.hpp"
+#include "market-timestamp-set.hpp"
#include "market.hpp"
#include "marketorderbook.hpp"
#include "monetaryamount.hpp"
#include "monetaryamountbycurrencyset.hpp"
#include "priceoptions.hpp"
#include "public-trade-vector.hpp"
+#include "time-window.hpp"
namespace cct {
+class AbstractMarketDataDeserializer;
+class AbstractMarketDataSerializer;
class CoincenterInfo;
class ExchangeConfig;
class FiatConverter;
@@ -40,7 +46,7 @@ class ExchangePublic : public ExchangeBase {
kWithPossibleFiatConversionAtExtremity
};
- virtual ~ExchangePublic() = default;
+ virtual ~ExchangePublic();
/// Check if public exchange is responding to basic health check, return true in this case.
/// Exchange that implements the HealthCheck do not need to add a retry mechanism.
@@ -97,14 +103,14 @@ class ExchangePublic : public ExchangeBase {
/// Retrieve the order book of given market.
/// It should be more precise that previous version with possibility to go deeper.
- virtual MarketOrderBook queryOrderBook(Market mk, int depth = kDefaultDepth) = 0;
+ MarketOrderBook getOrderBook(Market mk, int depth = kDefaultDepth);
+
+ /// Retrieve an ordered vector of recent last trades
+ PublicTradeVector getLastTrades(Market mk, int nbTrades = kNbLastTradesDefault);
/// Retrieve the total volume exchange on given market in the last 24 hours.
virtual MonetaryAmount queryLast24hVolume(Market mk) = 0;
- /// Retrieve an ordered vector of recent last trades
- virtual PublicTradeVector queryLastTrades(Market mk, int nbTrades = kNbLastTradesDefault) = 0;
-
/// Retrieve the last price of given market.
virtual MonetaryAmount queryLastPrice(Market mk) = 0;
@@ -176,7 +182,22 @@ class ExchangePublic : public ExchangeBase {
/// If no data found, return a 0 MonetaryAmount on given currency.
MonetaryAmount queryWithdrawalFeeOrZero(CurrencyCode currencyCode);
+ MarketTimestampSet pullMarketOrderBooksMarkets(TimeWindow timeWindow);
+
+ MarketTimestampSet pullTradeMarkets(TimeWindow timeWindow);
+
+ PublicTradeVector pullTradesForReplay(Market market, TimeWindow timeWindow);
+
+ MarketOrderBookVector pullMarketOrderBooksForReplay(Market market, TimeWindow timeWindow);
+
protected:
+ /// Retrieve the order book of given market.
+ /// It should be more precise that previous version with possibility to go deeper.
+ virtual MarketOrderBook queryOrderBook(Market mk, int depth = kDefaultDepth) = 0;
+
+ /// Retrieve an ordered vector of recent last trades
+ virtual PublicTradeVector queryLastTrades(Market mk, int nbTrades = kNbLastTradesDefault) = 0;
+
friend class ExchangePrivate;
ExchangePublic(std::string_view name, FiatConverter &fiatConverter, CommonAPI &commonApi,
@@ -188,7 +209,12 @@ class ExchangePublic : public ExchangeBase {
CommonAPI &_commonApi;
const CoincenterInfo &_coincenterInfo;
const ExchangeConfig &_exchangeConfig;
+ std::unique_ptr _marketDataDeserializerPtr;
+ std::unique_ptr _marketDataSerializerPtr;
std::recursive_mutex _publicRequestsMutex;
+
+ private:
+ AbstractMarketDataSerializer &getMarketDataSerializer();
};
} // namespace api
} // namespace cct
diff --git a/src/api/common/src/exchangeprivateapi.cpp b/src/api/common/src/exchangeprivateapi.cpp
index e7621c9d..79ec508d 100644
--- a/src/api/common/src/exchangeprivateapi.cpp
+++ b/src/api/common/src/exchangeprivateapi.cpp
@@ -557,7 +557,7 @@ PlaceOrderInfo ExchangePrivate::placeOrderProcess(MonetaryAmount &from, Monetary
if (tradeInfo.options.isSimulation() && !isSimulatedOrderSupported()) {
if (exchangeConfig().placeSimulateRealOrder()) {
log::debug("Place simulate real order - price {} will be overriden", price);
- MarketOrderBook marketOrderbook = _exchangePublic.queryOrderBook(mk);
+ MarketOrderBook marketOrderbook = _exchangePublic.getOrderBook(mk);
price = isSell ? marketOrderbook.getHighestTheoreticalPrice() : marketOrderbook.getLowestTheoreticalPrice();
} else {
PlaceOrderInfo placeOrderInfo = computeSimulatedMatchedPlacedOrderInfo(volume, price, tradeInfo);
diff --git a/src/api/common/src/exchangepublicapi.cpp b/src/api/common/src/exchangepublicapi.cpp
index d22015db..a16bddc9 100644
--- a/src/api/common/src/exchangepublicapi.cpp
+++ b/src/api/common/src/exchangepublicapi.cpp
@@ -6,6 +6,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -25,21 +26,44 @@
#include "exchangeconfig.hpp"
#include "exchangepublicapitypes.hpp"
#include "fiatconverter.hpp"
+#include "market-timestamp-set.hpp"
#include "market.hpp"
#include "marketorderbook.hpp"
#include "monetaryamount.hpp"
#include "priceoptions.hpp"
#include "priceoptionsdef.hpp"
+#include "time-window.hpp"
+#include "timedef.hpp"
#include "unreachable.hpp"
+#ifdef CCT_ENABLE_PROTO
+#include "proto-market-data-deserializer.hpp"
+#include "proto-market-data-serializer.hpp"
+#else
+#include "dummy-market-data-deserializer.hpp"
+#include "dummy-market-data-serializer.hpp"
+#endif
+
namespace cct::api {
+
+#ifdef CCT_ENABLE_PROTO
+using MarketDataDeserializer = ProtoMarketDataDeserializer;
+using MarketDataSerializer = ProtoMarketDataSerializer;
+#else
+using MarketDataDeserializer = DummyMarketDataDeserializer;
+using MarketDataSerializer = DummyMarketDataSerializer;
+#endif
+
ExchangePublic::ExchangePublic(std::string_view name, FiatConverter &fiatConverter, CommonAPI &commonApi,
const CoincenterInfo &coincenterInfo)
: _name(name),
_fiatConverter(fiatConverter),
_commonApi(commonApi),
_coincenterInfo(coincenterInfo),
- _exchangeConfig(coincenterInfo.exchangeConfig(name)) {}
+ _exchangeConfig(coincenterInfo.exchangeConfig(name)),
+ _marketDataDeserializerPtr(new MarketDataDeserializer(coincenterInfo.dataDir(), name)) {}
+
+ExchangePublic::~ExchangePublic() = default;
std::optional ExchangePublic::convert(MonetaryAmount from, CurrencyCode toCurrency,
const MarketsPath &conversionPath, const CurrencyCodeSet &fiats,
@@ -279,7 +303,7 @@ ExchangePublic::CurrenciesPath ExchangePublic::findCurrenciesPath(CurrencyCode f
std::optional ExchangePublic::computeLimitOrderPrice(Market mk, CurrencyCode fromCurrencyCode,
const PriceOptions &priceOptions) {
const int depth = priceOptions.isRelativePrice() ? std::abs(priceOptions.relativePrice()) : 1;
- return queryOrderBook(mk, depth).computeLimitPrice(fromCurrencyCode, priceOptions);
+ return getOrderBook(mk, depth).computeLimitPrice(fromCurrencyCode, priceOptions);
}
std::optional ExchangePublic::computeAvgOrderPrice(Market mk, MonetaryAmount from,
@@ -293,7 +317,7 @@ std::optional ExchangePublic::computeAvgOrderPrice(Market mk, Mo
} else if (priceOptions.priceStrategy() == PriceStrategy::kTaker) {
depth = kDefaultDepth;
}
- return queryOrderBook(mk, depth).computeAvgPrice(from, priceOptions);
+ return getOrderBook(mk, depth).computeAvgPrice(from, priceOptions);
}
std::optional ExchangePublic::RetrieveMarket(CurrencyCode c1, CurrencyCode c2, const MarketSet &markets) {
@@ -434,4 +458,62 @@ MonetaryAmount ExchangePublic::queryWithdrawalFeeOrZero(CurrencyCode currencyCod
return withdrawFee;
}
+MarketOrderBook ExchangePublic::getOrderBook(Market mk, int depth) {
+ std::lock_guard guard(_publicRequestsMutex);
+ const auto marketOrderBook = queryOrderBook(mk, depth);
+
+ if (_exchangeConfig.withMarketDataSerialization()) {
+ getMarketDataSerializer().push(marketOrderBook);
+ }
+ return marketOrderBook;
+}
+
+/// Retrieve an ordered vector of recent last trades
+PublicTradeVector ExchangePublic::getLastTrades(Market mk, int nbTrades) {
+ std::lock_guard guard(_publicRequestsMutex);
+ const auto lastTrades = queryLastTrades(mk, nbTrades);
+
+ if (_exchangeConfig.withMarketDataSerialization()) {
+ getMarketDataSerializer().push(mk, lastTrades);
+ }
+ return lastTrades;
+}
+
+MarketTimestampSet ExchangePublic::pullMarketOrderBooksMarkets(TimeWindow timeWindow) {
+ return _marketDataDeserializerPtr->pullMarketOrderBooksMarkets(timeWindow);
+}
+
+MarketTimestampSet ExchangePublic::pullTradeMarkets(TimeWindow timeWindow) {
+ return _marketDataDeserializerPtr->pullTradeMarkets(timeWindow);
+}
+
+PublicTradeVector ExchangePublic::pullTradesForReplay(Market market, TimeWindow timeWindow) {
+ return _marketDataDeserializerPtr->pullTrades(market, timeWindow);
+}
+
+MarketOrderBookVector ExchangePublic::pullMarketOrderBooksForReplay(Market market, TimeWindow timeWindow) {
+ return _marketDataDeserializerPtr->pullMarketOrderBooks(market, timeWindow);
+}
+
+AbstractMarketDataSerializer &ExchangePublic::getMarketDataSerializer() {
+ if (_marketDataSerializerPtr) {
+ return *_marketDataSerializerPtr;
+ }
+
+ const auto nowTime = Clock::now();
+
+ // Heuristic: load up to 1 week of data to retrieve the youngest written timestamp.
+ // This will be used in order not to write duplicate objects at the start of a new program after that a previous
+ // program run was stopped.
+ const TimeWindow largeTimeWindow{nowTime - std::chrono::weeks{1}, nowTime};
+
+ const MarketTimestampSets marketTimestampSets{pullMarketOrderBooksMarkets(largeTimeWindow),
+ pullTradeMarkets(largeTimeWindow)};
+
+ _marketDataSerializerPtr =
+ std::make_unique(_coincenterInfo.dataDir(), marketTimestampSets, name());
+
+ return *_marketDataSerializerPtr;
+}
+
} // namespace cct::api
diff --git a/src/api/exchanges/src/binancepublicapi.cpp b/src/api/exchanges/src/binancepublicapi.cpp
index 12516dae..c4ede616 100644
--- a/src/api/exchanges/src/binancepublicapi.cpp
+++ b/src/api/exchanges/src/binancepublicapi.cpp
@@ -238,7 +238,7 @@ MonetaryAmount BinancePublic::sanitizePrice(Market mk, MonetaryAmount pri) {
MonetaryAmount BinancePublic::computePriceForNotional(Market mk, int avgPriceMins) {
if (avgPriceMins == 0) {
// price should be the last matched price
- PublicTradeVector lastTrades = queryLastTrades(mk, 1);
+ PublicTradeVector lastTrades = getLastTrades(mk, 1);
if (!lastTrades.empty()) {
return lastTrades.front().price();
}
diff --git a/src/api/exchanges/src/bithumbpublicapi.cpp b/src/api/exchanges/src/bithumbpublicapi.cpp
index ed21261f..1779a42c 100644
--- a/src/api/exchanges/src/bithumbpublicapi.cpp
+++ b/src/api/exchanges/src/bithumbpublicapi.cpp
@@ -136,7 +136,7 @@ std::optional BithumbPublic::queryWithdrawalFee(CurrencyCode cur
MonetaryAmount BithumbPublic::queryLastPrice(Market mk) {
// Bithumb does not have a REST API endpoint for last price, let's compute it from the orderbook
- std::optional avgPrice = queryOrderBook(mk).averagePrice();
+ std::optional avgPrice = getOrderBook(mk).averagePrice();
if (!avgPrice) {
log::error("Empty order book for {} on {} cannot compute average price", mk, _name);
return MonetaryAmount(0, mk.quote());
diff --git a/src/api/interface/include/exchange.hpp b/src/api/interface/include/exchange.hpp
index a542430f..f6a5b12e 100644
--- a/src/api/interface/include/exchange.hpp
+++ b/src/api/interface/include/exchange.hpp
@@ -94,12 +94,12 @@ class Exchange {
return apiPublic().queryAllApproximatedOrderBooks(depth);
}
- MarketOrderBook queryOrderBook(Market mk, int depth = ExchangePublic::kDefaultDepth);
+ MarketOrderBook getOrderBook(Market mk, int depth = ExchangePublic::kDefaultDepth);
MonetaryAmount queryLast24hVolume(Market mk) { return apiPublic().queryLast24hVolume(mk); }
/// Retrieve an ordered vector of recent last trades
- PublicTradeVector queryLastTrades(Market mk, int nbTrades = ExchangePublic::kNbLastTradesDefault);
+ PublicTradeVector getLastTrades(Market mk, int nbTrades = ExchangePublic::kNbLastTradesDefault);
/// Retrieve the last price of given market.
MonetaryAmount queryLastPrice(Market mk) { return apiPublic().queryLastPrice(mk); }
diff --git a/src/api/interface/src/exchange.cpp b/src/api/interface/src/exchange.cpp
index 1763ae7e..f33f7a4c 100644
--- a/src/api/interface/src/exchange.cpp
+++ b/src/api/interface/src/exchange.cpp
@@ -48,12 +48,10 @@ bool Exchange::canDeposit(CurrencyCode currencyCode, const CurrencyExchangeFlatS
return lb->canDeposit();
}
-MarketOrderBook Exchange::queryOrderBook(Market mk, int depth) { return apiPublic().queryOrderBook(mk, depth); }
+MarketOrderBook Exchange::getOrderBook(Market mk, int depth) { return apiPublic().getOrderBook(mk, depth); }
/// Retrieve an ordered vector of recent last trades
-PublicTradeVector Exchange::queryLastTrades(Market mk, int nbTrades) {
- return apiPublic().queryLastTrades(mk, nbTrades);
-}
+PublicTradeVector Exchange::getLastTrades(Market mk, int nbTrades) { return apiPublic().getLastTrades(mk, nbTrades); }
void Exchange::updateCacheFile() const {
apiPublic().updateCacheFile();
diff --git a/src/engine/CMakeLists.txt b/src/engine/CMakeLists.txt
index f6191775..791f7987 100644
--- a/src/engine/CMakeLists.txt
+++ b/src/engine/CMakeLists.txt
@@ -6,6 +6,8 @@ target_link_libraries(coincenter_engine PUBLIC coincenter_api-common)
target_link_libraries(coincenter_engine PUBLIC coincenter_api-exchange)
target_link_libraries(coincenter_engine PUBLIC coincenter_api-interface)
target_link_libraries(coincenter_engine PUBLIC coincenter_objects)
+target_link_libraries(coincenter_engine PUBLIC coincenter_trading-algorithms)
+target_link_libraries(coincenter_engine PUBLIC coincenter_trading-common)
target_include_directories(coincenter_engine PUBLIC include)
add_unit_test(
@@ -81,6 +83,13 @@ add_unit_test(
../api/common/test/include
)
+add_unit_test(
+ replay-algorithm-name-iterator_test
+ test/replay-algorithm-name-iterator_test.cpp
+ LIBRARIES
+ coincenter_engine
+)
+
add_unit_test(
stringoptionparser_test
test/stringoptionparser_test.cpp
diff --git a/src/engine/include/coincenter-commands-iterator.hpp b/src/engine/include/coincenter-commands-iterator.hpp
new file mode 100644
index 00000000..809037dd
--- /dev/null
+++ b/src/engine/include/coincenter-commands-iterator.hpp
@@ -0,0 +1,28 @@
+#pragma once
+
+#include
+
+#include "coincentercommand.hpp"
+
+namespace cct {
+
+class CoincenterCommandsIterator {
+ public:
+ using CoincenterCommandSpan = std::span;
+
+ /// Initializes a new iterator on all coincenter commands.
+ explicit CoincenterCommandsIterator(CoincenterCommandSpan commands = CoincenterCommandSpan()) noexcept;
+
+ /// Returns 'true' if this iterator has still some command groups.
+ bool hasNextCommandGroup() const;
+
+ /// Get next grouped commands and advance the iterator.
+ /// The grouped commands are guaranteed to have same type and make it possible to parallelize requests when possible.
+ CoincenterCommandSpan nextCommandGroup();
+
+ private:
+ CoincenterCommandSpan _commands;
+ CoincenterCommandSpan::size_type _pos;
+};
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/engine/include/coincenter.hpp b/src/engine/include/coincenter.hpp
index eecef086..3cc68998 100644
--- a/src/engine/include/coincenter.hpp
+++ b/src/engine/include/coincenter.hpp
@@ -4,6 +4,8 @@
#include
#include "apikeysprovider.hpp"
+#include "cct_const.hpp"
+#include "cct_fixedcapacityvector.hpp"
#include "coincenterinfo.hpp"
#include "commonapi.hpp"
#include "exchange-names.hpp"
@@ -11,14 +13,18 @@
#include "exchangepool.hpp"
#include "exchangesorchestrator.hpp"
#include "fiatconverter.hpp"
+#include "market-trader-engine.hpp"
+#include "market.hpp"
#include "metricsexporter.hpp"
#include "ordersconstraints.hpp"
#include "queryresultprinter.hpp"
#include "queryresulttypes.hpp"
+#include "replay-options.hpp"
#include "transferablecommandresult.hpp"
namespace cct {
+class AbstractMarketTraderFactory;
class CoincenterCommand;
class CoincenterCommands;
class TradeOptions;
@@ -30,6 +36,7 @@ class Coincenter {
Coincenter(const CoincenterInfo &coincenterInfo, const ExchangeSecretsInfo &exchangeSecretsInfo);
+ /// Launch given commands and return the number of processed commands.
int process(const CoincenterCommands &coincenterCommands);
ExchangeHealthCheckStatus healthCheck(ExchangeNameSpan exchangeNames);
@@ -50,6 +57,10 @@ class Coincenter {
CurrencyCode equiCurrencyCode,
std::optional depth = std::nullopt);
+ /// Query market data without returning it.
+ /// This method is especially useful for serialization and metric exports.
+ void queryMarketDataPerExchange(std::span marketPerPublicExchange);
+
/// Retrieve the last 24h traded volume for exchanges supporting given market.
MonetaryAmountPerExchange getLast24hTradedVolumePerExchange(Market mk, ExchangeNameSpan exchangeNames);
@@ -132,6 +143,16 @@ class Coincenter {
const ExchangeName &toPrivateExchangeName,
const WithdrawOptions &withdrawOptions);
+ /// Retrieves the markets available for replay for exchanges selection that has some data during the last
+ /// 'replayDuration' time (so within the time frame [now - replayDuration, now])
+ MarketTimestampSetsPerExchange getMarketsAvailableForReplay(const ReplayOptions &replayOptions,
+ ExchangeNameSpan exchangeNames);
+
+ /// Replay all markets for exchanges selection that has some data during the last
+ /// 'replayDuration' time (so within the time frame [now - replayDuration, now])
+ void replay(const AbstractMarketTraderFactory &marketTraderFactory, const ReplayOptions &replayOptions, Market market,
+ ExchangeNameSpan exchangeNames);
+
/// Dumps the content of all file caches in data directory to save cURL queries.
void updateFileCaches() const;
@@ -147,8 +168,23 @@ class Coincenter {
const FiatConverter &fiatConverter() const { return _fiatConverter; }
private:
- TransferableCommandResultVector processCommand(
- const CoincenterCommand &cmd, std::span previousTransferableResults);
+ TransferableCommandResultVector processGroupedCommands(
+ std::span groupedCommands,
+ std::span previousTransferableResults);
+
+ using MarketTraderEngineVector = FixedCapacityVector;
+
+ void replayAlgorithm(const AbstractMarketTraderFactory &marketTraderFactory, std::string_view algorithmName,
+ const ReplayOptions &replayOptions, std::span marketTraderEngines,
+ const PublicExchangeNameVector &exchangesWithThisMarketData);
+
+ // TODO: may be moved somewhere else?
+ MarketTraderEngineVector createMarketTraderEngines(const ReplayOptions &replayOptions, Market market,
+ PublicExchangeNameVector &exchangesWithThisMarketData);
+
+ MarketTradeRangeStatsPerExchange tradingProcess(const ReplayOptions &replayOptions,
+ std::span marketTraderEngines,
+ ExchangeNameSpan exchangesWithThisMarketData);
const CoincenterInfo &_coincenterInfo;
api::CommonAPI _commonAPI;
diff --git a/src/engine/include/coincentercommand.hpp b/src/engine/include/coincentercommand.hpp
index df683d1f..db0894d1 100644
--- a/src/engine/include/coincentercommand.hpp
+++ b/src/engine/include/coincentercommand.hpp
@@ -13,6 +13,7 @@
#include "market.hpp"
#include "monetaryamount.hpp"
#include "ordersconstraints.hpp"
+#include "replay-options.hpp"
#include "tradeoptions.hpp"
#include "withdrawoptions.hpp"
#include "withdrawsconstraints.hpp"
@@ -44,6 +45,8 @@ class CoincenterCommand {
CoincenterCommand& setCur1(CurrencyCode cur1);
CoincenterCommand& setCur2(CurrencyCode cur2);
+ CoincenterCommand& setReplayOptions(ReplayOptions replayOptions);
+
CoincenterCommand& setPercentageAmount(bool value = true);
CoincenterCommand& withBalanceInUse(bool value = true);
@@ -74,16 +77,19 @@ class CoincenterCommand {
bool isPercentageAmount() const { return _isPercentageAmount; }
bool withBalanceInUse() const { return _withBalanceInUse; }
+ const ReplayOptions& replayOptions() const { return std::get(_specialOptions); }
+
bool operator==(const CoincenterCommand&) const noexcept = default;
using trivially_relocatable =
std::bool_constant && is_trivially_relocatable_v &&
is_trivially_relocatable_v &&
- is_trivially_relocatable_v && is_trivially_relocatable_v>::type;
+ is_trivially_relocatable_v && is_trivially_relocatable_v &&
+ is_trivially_relocatable_v>::type;
private:
- using SpecialOptions =
- std::variant;
+ using SpecialOptions = std::variant;
ExchangeNames _exchangeNames;
SpecialOptions _specialOptions;
diff --git a/src/engine/include/coincenteroptions.hpp b/src/engine/include/coincenteroptions.hpp
index eaa66bfb..24b44af3 100644
--- a/src/engine/include/coincenteroptions.hpp
+++ b/src/engine/include/coincenteroptions.hpp
@@ -9,7 +9,7 @@
#include "coincentercommandtype.hpp"
#include "coincenteroptionsdef.hpp"
#include "commandlineoption.hpp"
-#include "exchangepublicapi.hpp"
+#include "replay-options.hpp"
#include "timedef.hpp"
#include "tradedefinitions.hpp"
#include "tradeoptions.hpp"
@@ -30,6 +30,8 @@ class CoincenterCmdLineOptions {
TradeOptions computeTradeOptions() const;
WithdrawOptions computeWithdrawOptions() const;
+ ReplayOptions computeReplayOptions(Duration dur) const;
+
std::string_view getDataDir() const { return dataDir.empty() ? SelectDefaultDataDir() : dataDir; }
std::pair getTradeArgStr() const;
@@ -100,6 +102,13 @@ class CoincenterCmdLineOptions {
std::string_view lastTrades;
+ std::string_view marketData;
+
+ std::optional replay;
+ std::string_view algorithmNames;
+ std::string_view market;
+ std::optional replayMarkets;
+
CommandLineOptionalInt32 repeats;
int32_t monitoringPort = CoincenterCmdLineOptionsDefinitions::kDefaultMonitoringPort;
int32_t depth = kUndefinedDepth;
@@ -114,6 +123,8 @@ class CoincenterCmdLineOptions {
bool version = false;
bool useMonitoring = false;
bool withBalanceInUse = false;
+ bool validate = false;
+ bool validateOnly = false;
bool operator==(const CoincenterCmdLineOptions&) const noexcept = default;
diff --git a/src/engine/include/coincenteroptionsdef.hpp b/src/engine/include/coincenteroptionsdef.hpp
index a27f1efc..aeb4c8e1 100644
--- a/src/engine/include/coincenteroptionsdef.hpp
+++ b/src/engine/include/coincenteroptionsdef.hpp
@@ -432,6 +432,50 @@ struct CoincenterAllowedOptions : private CoincenterCmdLineOptionsDefinitions {
"Prints withdraw fees for matching currency and exchanges.\n"
"Currency and exchanges are optional, if specified, output will be filtered to match them."},
&OptValueType::withdrawFees},
+ {{{"Automation", 8000},
+ "market-data",
+ "",
+ "Query last trades and order books of given market without printing the result on screen, for given exchanges "
+ "if specified.\n"
+ "This is the equivalent of calling last-trades and order-book but is useful combined with the repeat "
+ "command to store market data on disk."},
+ &OptValueType::marketData},
+ {{{"Automation", 8000},
+ "replay-markets",
+ "",
+ "Print markets available for replay, that is, markets that have some data within the time "
+ "window {now - duration, now}."},
+ &OptValueType::replayMarkets},
+ {{{"Automation", 8001},
+ "replay",
+ "",
+ "Replay algorithms on serialized, historical data."
+ "\nAll known algorithms will be replayed one by one, on all stored markets that have some data within the time "
+ "window {now - duration, now}. Use below flags to filter more precisely on which data to replay from."},
+ &OptValueType::replay},
+ {{{"Automation", 8002},
+ "--algorithms",
+ "",
+ "Pick specific algorithm names to replay with. Default will replay with all known ones."},
+ &OptValueType::algorithmNames},
+ {{{"Automation", 8003},
+ "--market",
+ "",
+ "Only replay for specific market. Default will replay all stored markets."},
+ &OptValueType::market},
+ {{{"Automation", 8003},
+ "--validate",
+ "",
+ "Filter invalid data during replay.\nThis is disabled by default, use this option when you suspect that "
+ "invalid data may be present in the replayed time window."},
+ &OptValueType::validate},
+ {{{"Automation", 8003},
+ "--validate-only",
+ "",
+ "Instead of launching replay algorithm, only validates serialized data."
+ "\nNominal replay will not validate input data to optimize performance, use this option to validate data once "
+ "and for all."},
+ &OptValueType::validateOnly},
{{{"Monitoring", 9000},
"--monitoring",
"",
diff --git a/src/engine/include/exchangesorchestrator.hpp b/src/engine/include/exchangesorchestrator.hpp
index a2c052dd..4498a709 100644
--- a/src/engine/include/exchangesorchestrator.hpp
+++ b/src/engine/include/exchangesorchestrator.hpp
@@ -6,14 +6,18 @@
#include "exchange-names.hpp"
#include "exchangename.hpp"
#include "exchangeretriever.hpp"
+#include "market-trader-engine.hpp"
#include "market.hpp"
#include "queryresulttypes.hpp"
#include "threadpool.hpp"
+#include "time-window.hpp"
#include "withdrawoptions.hpp"
namespace cct {
+class ReplayOptions;
class RequestsConfig;
+
class ExchangesOrchestrator {
public:
using UniquePublicSelectedExchanges = ExchangeRetriever::UniquePublicSelectedExchanges;
@@ -89,6 +93,19 @@ class ExchangesOrchestrator {
MonetaryAmountPerExchange getLastPricePerExchange(Market mk, ExchangeNameSpan exchangeNames);
+ MarketDataPerExchange getMarketDataPerExchange(std::span marketPerPublicExchange,
+ ExchangeNameSpan exchangeNames);
+
+ MarketTimestampSetsPerExchange pullAvailableMarketsForReplay(TimeWindow timeWindow, ExchangeNameSpan exchangeNames);
+
+ MarketTradeRangeStatsPerExchange traderConsumeRange(const ReplayOptions &replayOptions, TimeWindow subTimeWindow,
+ std::span marketTraderEngines,
+ ExchangeNameSpan exchangeNames);
+
+ MarketTradingGlobalResultPerExchange getMarketTraderResultPerExchange(
+ std::span marketTraderEngines, MarketTradeRangeStatsPerExchange &&tradeRangeStatsPerExchange,
+ ExchangeNameSpan exchangeNames);
+
private:
ExchangeRetriever _exchangeRetriever;
ThreadPool _threadPool;
diff --git a/src/engine/include/query-result-type-helpers.hpp b/src/engine/include/query-result-type-helpers.hpp
new file mode 100644
index 00000000..6db39046
--- /dev/null
+++ b/src/engine/include/query-result-type-helpers.hpp
@@ -0,0 +1,16 @@
+#pragma once
+
+#include "exchangepublicapitypes.hpp"
+#include "market-timestamp-set.hpp"
+#include "market.hpp"
+#include "queryresulttypes.hpp"
+
+namespace cct {
+
+bool ContainsMarket(Market market, const MarketTimestampSet &marketTimestampSet);
+
+bool ContainsMarket(Market market, const MarketTimestampSets &marketTimestampSets);
+
+MarketSet ComputeAllMarkets(const MarketTimestampSetsPerExchange &marketTimestampSetsPerExchange);
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/engine/include/queryresultprinter.hpp b/src/engine/include/queryresultprinter.hpp
index 6c0a1f52..81e692ac 100644
--- a/src/engine/include/queryresultprinter.hpp
+++ b/src/engine/include/queryresultprinter.hpp
@@ -15,6 +15,7 @@
#include "ordersconstraints.hpp"
#include "queryresulttypes.hpp"
#include "simpletable.hpp"
+#include "time-window.hpp"
#include "withdrawsconstraints.hpp"
namespace cct {
@@ -66,7 +67,7 @@ class QueryResultPrinter {
}
void printClosedOrders(const ClosedOrdersPerExchange &closedOrdersPerExchange,
- const OrdersConstraints &ordersConstraints) const;
+ const OrdersConstraints &ordersConstraints = OrdersConstraints{}) const;
void printOpenedOrders(const OpenedOrdersPerExchange &openedOrdersPerExchange,
const OrdersConstraints &ordersConstraints) const;
@@ -104,6 +105,13 @@ class QueryResultPrinter {
const TradedAmountsVectorWithFinalAmountPerExchange &tradedAmountsVectorWithFinalAmountPerExchange,
CurrencyCode currencyCode) const;
+ void printMarketsForReplay(TimeWindow timeWindow,
+ const MarketTimestampSetsPerExchange &marketTimestampSetsPerExchange);
+
+ void printMarketTradingResults(TimeWindow timeWindow,
+ const MarketTradingGlobalResultPerExchange &marketTradingResultPerExchange,
+ CoincenterCommandType commandType) const;
+
private:
void printTrades(const TradeResultPerExchange &tradeResultPerExchange, MonetaryAmount amount, bool isPercentageTrade,
CurrencyCode toCurrency, const TradeOptions &tradeOptions, CoincenterCommandType commandType) const;
diff --git a/src/engine/include/queryresulttypes.hpp b/src/engine/include/queryresulttypes.hpp
index 60f1028b..5af47573 100644
--- a/src/engine/include/queryresulttypes.hpp
+++ b/src/engine/include/queryresulttypes.hpp
@@ -13,10 +13,14 @@
#include "currencyexchangeflatset.hpp"
#include "exchangeprivateapitypes.hpp"
#include "exchangepublicapitypes.hpp"
+#include "market-timestamp-set.hpp"
+#include "market-trading-global-result.hpp"
+#include "market-trading-result.hpp"
#include "marketorderbook.hpp"
#include "monetaryamount.hpp"
#include "monetaryamountbycurrencyset.hpp"
#include "public-trade-vector.hpp"
+#include "trade-range-stats.hpp"
#include "traderesult.hpp"
#include "wallet.hpp"
#include "withdrawinfo.hpp"
@@ -31,6 +35,8 @@ using MarketOrderBookConversionRate = std::tuple;
+using MarketPerExchange = FixedCapacityVector, kNbSupportedExchanges>;
+
using MarketsPerExchange = FixedCapacityVector, kNbSupportedExchanges>;
using MonetaryAmountPerExchange = FixedCapacityVector, kNbSupportedExchanges>;
@@ -40,6 +46,9 @@ using MonetaryAmountByCurrencySetPerExchange =
using TradesPerExchange = FixedCapacityVector, kNbSupportedExchanges>;
+using MarketDataPerExchange =
+ FixedCapacityVector>, kNbSupportedExchanges>;
+
using TradeResultPerExchange = SmallVector, kTypicalNbPrivateAccounts>;
using TradedAmountsVectorWithFinalAmountPerExchange =
@@ -68,4 +77,14 @@ using DeliveredWithdrawInfoWithExchanges = std::pair, kTypicalNbPrivateAccounts>;
using ConversionPathPerExchange = FixedCapacityVector, kNbSupportedExchanges>;
+
+using MarketTimestampSetsPerExchange = FixedCapacityVector, kNbSupportedExchanges>;
+
+using MarketTradeRangeStatsPerExchange = FixedCapacityVector, kNbSupportedExchanges>;
+
+using MarketTradingResultPerExchange = FixedCapacityVector, kNbSupportedExchanges>;
+
+using MarketTradingGlobalResultPerExchange =
+ FixedCapacityVector, kNbSupportedExchanges>;
+
} // namespace cct
diff --git a/src/engine/include/replay-algorithm-name-iterator.hpp b/src/engine/include/replay-algorithm-name-iterator.hpp
new file mode 100644
index 00000000..21995286
--- /dev/null
+++ b/src/engine/include/replay-algorithm-name-iterator.hpp
@@ -0,0 +1,29 @@
+#pragma once
+
+#include
+#include
+#include
+
+namespace cct {
+
+/// Convenient class to iterate on the algorithm names, comma separated.
+/// If 'algorithmNames' is empty, it will loop on all available ones (given by 'allAlgorithms')
+class ReplayAlgorithmNameIterator {
+ public:
+ ReplayAlgorithmNameIterator(std::string_view algorithmNames, std::span allAlgorithms);
+
+ /// Returns true if and only if there is at least one additional algorithm name to iterate on.
+ bool hasNext() const;
+
+ /// Get next algorithm name and advance the iterator.
+ /// Undefined behavior if 'hasNext' is 'false'.
+ std::string_view next();
+
+ private:
+ std::span _allAlgorithms;
+ std::string_view _algorithmNames;
+ int32_t _begPos;
+ int32_t _endPos;
+};
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/engine/include/replay-options.hpp b/src/engine/include/replay-options.hpp
new file mode 100644
index 00000000..44a762a2
--- /dev/null
+++ b/src/engine/include/replay-options.hpp
@@ -0,0 +1,33 @@
+#pragma once
+
+#include
+#include
+
+#include "time-window.hpp"
+
+namespace cct {
+
+class ReplayOptions {
+ public:
+ enum class ReplayMode : int8_t { kValidateOnly, kCheckedLaunchAlgorithm, kUncheckedLaunchAlgorithm };
+
+ ReplayOptions() noexcept = default;
+
+ /// Algorithm names should be comma separated. Empty string will match all.
+ ReplayOptions(TimeWindow timeWindow, std::string_view algorithmNames, ReplayMode replayMode);
+
+ TimeWindow timeWindow() const { return _timeWindow; }
+
+ std::string_view algorithmNames() const;
+
+ ReplayMode replayMode() const { return _replayMode; }
+
+ bool operator==(const ReplayOptions &) const noexcept = default;
+
+ private:
+ TimeWindow _timeWindow;
+ std::string_view _algorithmNames;
+ ReplayMode _replayMode;
+};
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/engine/include/stringoptionparser.hpp b/src/engine/include/stringoptionparser.hpp
index 58bed922..330d3aeb 100644
--- a/src/engine/include/stringoptionparser.hpp
+++ b/src/engine/include/stringoptionparser.hpp
@@ -10,6 +10,7 @@
#include "exchange-names.hpp"
#include "market.hpp"
#include "monetaryamount.hpp"
+#include "timedef.hpp"
namespace cct {
class StringOptionParser {
@@ -29,6 +30,10 @@ class StringOptionParser {
/// after the currency
CurrencyCode parseCurrency(FieldIs fieldIs = FieldIs::kMandatory, char delimiter = ',');
+ /// If FieldIs is kOptional and there is no duration, kUndefinedDuration duration will be returned.
+ /// otherwise exception invalid_argument will be raised
+ Duration parseDuration(FieldIs fieldIs = FieldIs::kMandatory);
+
/// If FieldIs is kOptional and there is no market, default market will be returned.
/// otherwise exception invalid_argument will be raised.
/// @param delimiter defines the expected character (could be not present, which means end of parsing)
@@ -55,4 +60,4 @@ class StringOptionParser {
std::string_view _opt;
std::string_view::size_type _pos{};
};
-} // namespace cct
\ No newline at end of file
+} // namespace cct
diff --git a/src/engine/src/coincenter-commands-iterator.cpp b/src/engine/src/coincenter-commands-iterator.cpp
new file mode 100644
index 00000000..f0f773bd
--- /dev/null
+++ b/src/engine/src/coincenter-commands-iterator.cpp
@@ -0,0 +1,74 @@
+#include "coincenter-commands-iterator.hpp"
+
+#include
+
+#include "cct_const.hpp"
+#include "coincentercommandtype.hpp"
+#include "exchangename.hpp"
+
+namespace cct {
+
+CoincenterCommandsIterator::CoincenterCommandsIterator(CoincenterCommandSpan commands) noexcept
+ : _commands(commands), _pos() {}
+
+namespace {
+using PublicExchangePresenceBitset = std::bitset;
+
+bool UpdateBitsetAreNewExchanges(const CoincenterCommand &command,
+ PublicExchangePresenceBitset &publicExchangePresence) {
+ if (command.exchangeNames().empty()) {
+ // All public exchanges used
+ const auto result = publicExchangePresence.none();
+ publicExchangePresence.set();
+ return result;
+ }
+ for (const ExchangeName &exchangeName : command.exchangeNames()) {
+ const auto exchangePos = exchangeName.publicExchangePos();
+ if (publicExchangePresence[exchangePos]) {
+ return false;
+ }
+ publicExchangePresence.set(exchangePos);
+ }
+ return true;
+}
+
+bool CommandTypeCanBeGrouped(CoincenterCommandType type) {
+ // Compatible command types need to be explicitly set
+ // For now, only market data is compatible
+ switch (type) {
+ case CoincenterCommandType::kMarketData:
+ return true;
+ default:
+ return false;
+ }
+}
+
+} // namespace
+
+bool CoincenterCommandsIterator::hasNextCommandGroup() const { return _pos < _commands.size(); }
+
+CoincenterCommandsIterator::CoincenterCommandSpan CoincenterCommandsIterator::nextCommandGroup() {
+ CoincenterCommandSpan groupedCommands(_commands.begin() + _pos, 1U);
+
+ if (CommandTypeCanBeGrouped(groupedCommands.front().type())) {
+ PublicExchangePresenceBitset publicExchangePresence;
+ UpdateBitsetAreNewExchanges(groupedCommands.front(), publicExchangePresence);
+
+ while (_pos + groupedCommands.size() < _commands.size()) {
+ const CoincenterCommand &nextCommand = _commands[_pos + groupedCommands.size()];
+ if (nextCommand.type() != groupedCommands.front().type()) {
+ break;
+ }
+ if (!UpdateBitsetAreNewExchanges(nextCommand, publicExchangePresence)) {
+ break;
+ }
+ // Add new command to group
+ groupedCommands = CoincenterCommandSpan(groupedCommands.data(), groupedCommands.size() + 1);
+ }
+ }
+
+ _pos += groupedCommands.size();
+ return groupedCommands;
+}
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/engine/src/coincenter.cpp b/src/engine/src/coincenter.cpp
index 1a08e76e..8dbbfa08 100644
--- a/src/engine/src/coincenter.cpp
+++ b/src/engine/src/coincenter.cpp
@@ -13,6 +13,7 @@
#include "cct_exception.hpp"
#include "cct_invalid_argument_exception.hpp"
#include "cct_log.hpp"
+#include "coincenter-commands-iterator.hpp"
#include "coincentercommand.hpp"
#include "coincentercommands.hpp"
#include "coincentercommandtype.hpp"
@@ -25,11 +26,18 @@
#include "exchangepublicapi.hpp"
#include "exchangeretriever.hpp"
#include "exchangesecretsinfo.hpp"
+#include "market-timestamp-set.hpp"
+#include "market-trader-engine.hpp"
+#include "market-trader-factory.hpp"
#include "market.hpp"
#include "monetaryamount.hpp"
#include "ordersconstraints.hpp"
+#include "query-result-type-helpers.hpp"
#include "queryresultprinter.hpp"
#include "queryresulttypes.hpp"
+#include "replay-algorithm-name-iterator.hpp"
+#include "replay-options.hpp"
+#include "time-window.hpp"
#include "timedef.hpp"
#include "transferablecommandresult.hpp"
#include "withdrawsconstraints.hpp"
@@ -113,35 +121,40 @@ int Coincenter::process(const CoincenterCommands &coincenterCommands) {
}
}
TransferableCommandResultVector transferableResults;
- for (const auto &cmd : commands) {
- transferableResults = processCommand(cmd, transferableResults);
+ CoincenterCommandsIterator commandsIterator(commands);
+ while (commandsIterator.hasNextCommandGroup()) {
+ const auto groupedCommands = commandsIterator.nextCommandGroup();
+ transferableResults = processGroupedCommands(groupedCommands, transferableResults);
++nbCommandsProcessed;
}
}
return nbCommandsProcessed;
}
-TransferableCommandResultVector Coincenter::processCommand(
- const CoincenterCommand &cmd, std::span previousTransferableResults) {
+TransferableCommandResultVector Coincenter::processGroupedCommands(
+ std::span groupedCommands,
+ std::span previousTransferableResults) {
TransferableCommandResultVector transferableResults;
- switch (cmd.type()) {
+ const auto &firstCmd = groupedCommands.front();
+ // All grouped commands have same type - logic to handle multiple commands in a group should be handled per use case
+ switch (firstCmd.type()) {
case CoincenterCommandType::kHealthCheck: {
- const auto healthCheckStatus = healthCheck(cmd.exchangeNames());
+ const auto healthCheckStatus = healthCheck(firstCmd.exchangeNames());
_queryResultPrinter.printHealthCheck(healthCheckStatus);
break;
}
case CoincenterCommandType::kCurrencies: {
- const auto currenciesPerExchange = getCurrenciesPerExchange(cmd.exchangeNames());
+ const auto currenciesPerExchange = getCurrenciesPerExchange(firstCmd.exchangeNames());
_queryResultPrinter.printCurrencies(currenciesPerExchange);
break;
}
case CoincenterCommandType::kMarkets: {
- const auto marketsPerExchange = getMarketsPerExchange(cmd.cur1(), cmd.cur2(), cmd.exchangeNames());
- _queryResultPrinter.printMarkets(cmd.cur1(), cmd.cur2(), marketsPerExchange, cmd.type());
+ const auto marketsPerExchange = getMarketsPerExchange(firstCmd.cur1(), firstCmd.cur2(), firstCmd.exchangeNames());
+ _queryResultPrinter.printMarkets(firstCmd.cur1(), firstCmd.cur2(), marketsPerExchange, firstCmd.type());
break;
}
case CoincenterCommandType::kConversion: {
- if (cmd.amount().isDefault()) {
+ if (firstCmd.amount().isDefault()) {
std::array startAmountsPerExchangePos;
bool oneSet = false;
for (const auto &transferableResult : previousTransferableResults) {
@@ -158,90 +171,96 @@ TransferableCommandResultVector Coincenter::processCommand(
throw invalid_argument("Missing input amount to convert from");
}
- const auto conversionPerExchange = getConversion(startAmountsPerExchangePos, cmd.cur1(), cmd.exchangeNames());
- _queryResultPrinter.printConversion(startAmountsPerExchangePos, cmd.cur1(), conversionPerExchange);
+ const auto conversionPerExchange =
+ getConversion(startAmountsPerExchangePos, firstCmd.cur1(), firstCmd.exchangeNames());
+ _queryResultPrinter.printConversion(startAmountsPerExchangePos, firstCmd.cur1(), conversionPerExchange);
FillConversionTransferableCommandResults(conversionPerExchange, transferableResults);
} else {
- const auto conversionPerExchange = getConversion(cmd.amount(), cmd.cur1(), cmd.exchangeNames());
- _queryResultPrinter.printConversion(cmd.amount(), cmd.cur1(), conversionPerExchange);
+ const auto conversionPerExchange = getConversion(firstCmd.amount(), firstCmd.cur1(), firstCmd.exchangeNames());
+ _queryResultPrinter.printConversion(firstCmd.amount(), firstCmd.cur1(), conversionPerExchange);
FillConversionTransferableCommandResults(conversionPerExchange, transferableResults);
}
break;
}
case CoincenterCommandType::kConversionPath: {
- const auto conversionPathPerExchange = getConversionPaths(cmd.market(), cmd.exchangeNames());
- _queryResultPrinter.printConversionPath(cmd.market(), conversionPathPerExchange);
+ const auto conversionPathPerExchange = getConversionPaths(firstCmd.market(), firstCmd.exchangeNames());
+ _queryResultPrinter.printConversionPath(firstCmd.market(), conversionPathPerExchange);
break;
}
case CoincenterCommandType::kLastPrice: {
- const auto lastPricePerExchange = getLastPricePerExchange(cmd.market(), cmd.exchangeNames());
- _queryResultPrinter.printLastPrice(cmd.market(), lastPricePerExchange);
+ const auto lastPricePerExchange = getLastPricePerExchange(firstCmd.market(), firstCmd.exchangeNames());
+ _queryResultPrinter.printLastPrice(firstCmd.market(), lastPricePerExchange);
break;
}
case CoincenterCommandType::kTicker: {
- const auto exchangeTickerMaps = getTickerInformation(cmd.exchangeNames());
+ const auto exchangeTickerMaps = getTickerInformation(firstCmd.exchangeNames());
_queryResultPrinter.printTickerInformation(exchangeTickerMaps);
break;
}
case CoincenterCommandType::kOrderbook: {
const auto marketOrderBooksConversionRates =
- getMarketOrderBooks(cmd.market(), cmd.exchangeNames(), cmd.cur1(), cmd.optDepth());
- _queryResultPrinter.printMarketOrderBooks(cmd.market(), cmd.cur1(), cmd.optDepth(),
+ getMarketOrderBooks(firstCmd.market(), firstCmd.exchangeNames(), firstCmd.cur1(), firstCmd.optDepth());
+ _queryResultPrinter.printMarketOrderBooks(firstCmd.market(), firstCmd.cur1(), firstCmd.optDepth(),
marketOrderBooksConversionRates);
break;
}
case CoincenterCommandType::kLastTrades: {
- const auto lastTradesPerExchange = getLastTradesPerExchange(cmd.market(), cmd.exchangeNames(), cmd.optDepth());
- _queryResultPrinter.printLastTrades(cmd.market(), cmd.optDepth(), lastTradesPerExchange);
+ const auto lastTradesPerExchange =
+ getLastTradesPerExchange(firstCmd.market(), firstCmd.exchangeNames(), firstCmd.optDepth());
+ _queryResultPrinter.printLastTrades(firstCmd.market(), firstCmd.optDepth(), lastTradesPerExchange);
break;
}
case CoincenterCommandType::kLast24hTradedVolume: {
- const auto tradedVolumePerExchange = getLast24hTradedVolumePerExchange(cmd.market(), cmd.exchangeNames());
- _queryResultPrinter.printLast24hTradedVolume(cmd.market(), tradedVolumePerExchange);
+ const auto tradedVolumePerExchange =
+ getLast24hTradedVolumePerExchange(firstCmd.market(), firstCmd.exchangeNames());
+ _queryResultPrinter.printLast24hTradedVolume(firstCmd.market(), tradedVolumePerExchange);
break;
}
case CoincenterCommandType::kWithdrawFees: {
- const auto withdrawFeesPerExchange = getWithdrawFees(cmd.cur1(), cmd.exchangeNames());
- _queryResultPrinter.printWithdrawFees(withdrawFeesPerExchange, cmd.cur1());
+ const auto withdrawFeesPerExchange = getWithdrawFees(firstCmd.cur1(), firstCmd.exchangeNames());
+ _queryResultPrinter.printWithdrawFees(withdrawFeesPerExchange, firstCmd.cur1());
break;
}
case CoincenterCommandType::kBalance: {
- const auto amountIncludePolicy = cmd.withBalanceInUse() ? BalanceOptions::AmountIncludePolicy::kWithBalanceInUse
- : BalanceOptions::AmountIncludePolicy::kOnlyAvailable;
- const BalanceOptions balanceOptions(amountIncludePolicy, cmd.cur1());
- const auto balancePerExchange = getBalance(cmd.exchangeNames(), balanceOptions);
- _queryResultPrinter.printBalance(balancePerExchange, cmd.cur1());
+ const auto amountIncludePolicy = firstCmd.withBalanceInUse()
+ ? BalanceOptions::AmountIncludePolicy::kWithBalanceInUse
+ : BalanceOptions::AmountIncludePolicy::kOnlyAvailable;
+ const BalanceOptions balanceOptions(amountIncludePolicy, firstCmd.cur1());
+ const auto balancePerExchange = getBalance(firstCmd.exchangeNames(), balanceOptions);
+ _queryResultPrinter.printBalance(balancePerExchange, firstCmd.cur1());
break;
}
case CoincenterCommandType::kDepositInfo: {
- const auto walletPerExchange = getDepositInfo(cmd.exchangeNames(), cmd.cur1());
- _queryResultPrinter.printDepositInfo(cmd.cur1(), walletPerExchange);
+ const auto walletPerExchange = getDepositInfo(firstCmd.exchangeNames(), firstCmd.cur1());
+ _queryResultPrinter.printDepositInfo(firstCmd.cur1(), walletPerExchange);
break;
}
case CoincenterCommandType::kOrdersClosed: {
- const auto closedOrdersPerExchange = getClosedOrders(cmd.exchangeNames(), cmd.ordersConstraints());
- _queryResultPrinter.printClosedOrders(closedOrdersPerExchange, cmd.ordersConstraints());
+ const auto closedOrdersPerExchange = getClosedOrders(firstCmd.exchangeNames(), firstCmd.ordersConstraints());
+ _queryResultPrinter.printClosedOrders(closedOrdersPerExchange, firstCmd.ordersConstraints());
break;
}
case CoincenterCommandType::kOrdersOpened: {
- const auto openedOrdersPerExchange = getOpenedOrders(cmd.exchangeNames(), cmd.ordersConstraints());
- _queryResultPrinter.printOpenedOrders(openedOrdersPerExchange, cmd.ordersConstraints());
+ const auto openedOrdersPerExchange = getOpenedOrders(firstCmd.exchangeNames(), firstCmd.ordersConstraints());
+ _queryResultPrinter.printOpenedOrders(openedOrdersPerExchange, firstCmd.ordersConstraints());
break;
}
case CoincenterCommandType::kOrdersCancel: {
- const auto nbCancelledOrdersPerExchange = cancelOrders(cmd.exchangeNames(), cmd.ordersConstraints());
- _queryResultPrinter.printCancelledOrders(nbCancelledOrdersPerExchange, cmd.ordersConstraints());
+ const auto nbCancelledOrdersPerExchange = cancelOrders(firstCmd.exchangeNames(), firstCmd.ordersConstraints());
+ _queryResultPrinter.printCancelledOrders(nbCancelledOrdersPerExchange, firstCmd.ordersConstraints());
break;
}
case CoincenterCommandType::kRecentDeposits: {
- const auto depositsPerExchange = getRecentDeposits(cmd.exchangeNames(), cmd.withdrawsOrDepositsConstraints());
- _queryResultPrinter.printRecentDeposits(depositsPerExchange, cmd.withdrawsOrDepositsConstraints());
+ const auto depositsPerExchange =
+ getRecentDeposits(firstCmd.exchangeNames(), firstCmd.withdrawsOrDepositsConstraints());
+ _queryResultPrinter.printRecentDeposits(depositsPerExchange, firstCmd.withdrawsOrDepositsConstraints());
break;
}
case CoincenterCommandType::kRecentWithdraws: {
- const auto withdrawsPerExchange = getRecentWithdraws(cmd.exchangeNames(), cmd.withdrawsOrDepositsConstraints());
- _queryResultPrinter.printRecentWithdraws(withdrawsPerExchange, cmd.withdrawsOrDepositsConstraints());
+ const auto withdrawsPerExchange =
+ getRecentWithdraws(firstCmd.exchangeNames(), firstCmd.withdrawsOrDepositsConstraints());
+ _queryResultPrinter.printRecentWithdraws(withdrawsPerExchange, firstCmd.withdrawsOrDepositsConstraints());
break;
}
case CoincenterCommandType::kTrade: {
@@ -249,50 +268,83 @@ TransferableCommandResultVector Coincenter::processCommand(
// - standard full information with an amount to trade, a destination currency and an optional list of exchanges
// where to trade
// - a currency - the destination one, and start amount and exchange(s) should come from previous command result
- auto [startAmount, exchangeNames] = ComputeTradeAmountAndExchanges(cmd, previousTransferableResults);
+ auto [startAmount, exchangeNames] = ComputeTradeAmountAndExchanges(firstCmd, previousTransferableResults);
if (startAmount.isDefault()) {
break;
}
const auto tradeResultPerExchange =
- trade(startAmount, cmd.isPercentageAmount(), cmd.cur1(), exchangeNames, cmd.tradeOptions());
- _queryResultPrinter.printTrades(tradeResultPerExchange, startAmount, cmd.isPercentageAmount(), cmd.cur1(),
- cmd.tradeOptions());
+ trade(startAmount, firstCmd.isPercentageAmount(), firstCmd.cur1(), exchangeNames, firstCmd.tradeOptions());
+ _queryResultPrinter.printTrades(tradeResultPerExchange, startAmount, firstCmd.isPercentageAmount(),
+ firstCmd.cur1(), firstCmd.tradeOptions());
FillTradeTransferableCommandResults(tradeResultPerExchange, transferableResults);
break;
}
case CoincenterCommandType::kBuy: {
- const auto tradeResultPerExchange = smartBuy(cmd.amount(), cmd.exchangeNames(), cmd.tradeOptions());
- _queryResultPrinter.printBuyTrades(tradeResultPerExchange, cmd.amount(), cmd.tradeOptions());
+ const auto tradeResultPerExchange =
+ smartBuy(firstCmd.amount(), firstCmd.exchangeNames(), firstCmd.tradeOptions());
+ _queryResultPrinter.printBuyTrades(tradeResultPerExchange, firstCmd.amount(), firstCmd.tradeOptions());
FillTradeTransferableCommandResults(tradeResultPerExchange, transferableResults);
break;
}
case CoincenterCommandType::kSell: {
- auto [startAmount, exchangeNames] = ComputeTradeAmountAndExchanges(cmd, previousTransferableResults);
+ auto [startAmount, exchangeNames] = ComputeTradeAmountAndExchanges(firstCmd, previousTransferableResults);
if (startAmount.isDefault()) {
break;
}
const auto tradeResultPerExchange =
- smartSell(startAmount, cmd.isPercentageAmount(), exchangeNames, cmd.tradeOptions());
- _queryResultPrinter.printSellTrades(tradeResultPerExchange, cmd.amount(), cmd.isPercentageAmount(),
- cmd.tradeOptions());
+ smartSell(startAmount, firstCmd.isPercentageAmount(), exchangeNames, firstCmd.tradeOptions());
+ _queryResultPrinter.printSellTrades(tradeResultPerExchange, firstCmd.amount(), firstCmd.isPercentageAmount(),
+ firstCmd.tradeOptions());
FillTradeTransferableCommandResults(tradeResultPerExchange, transferableResults);
break;
}
case CoincenterCommandType::kWithdrawApply: {
- const auto [grossAmount, exchangeName] = ComputeWithdrawAmount(cmd, previousTransferableResults);
+ const auto [grossAmount, exchangeName] = ComputeWithdrawAmount(firstCmd, previousTransferableResults);
if (grossAmount.isDefault()) {
break;
}
- const auto deliveredWithdrawInfoWithExchanges = withdraw(grossAmount, cmd.isPercentageAmount(), exchangeName,
- cmd.exchangeNames().back(), cmd.withdrawOptions());
- _queryResultPrinter.printWithdraw(deliveredWithdrawInfoWithExchanges, cmd.isPercentageAmount(),
- cmd.withdrawOptions());
+ const auto deliveredWithdrawInfoWithExchanges =
+ withdraw(grossAmount, firstCmd.isPercentageAmount(), exchangeName, firstCmd.exchangeNames().back(),
+ firstCmd.withdrawOptions());
+ _queryResultPrinter.printWithdraw(deliveredWithdrawInfoWithExchanges, firstCmd.isPercentageAmount(),
+ firstCmd.withdrawOptions());
transferableResults.emplace_back(deliveredWithdrawInfoWithExchanges.first[1]->createExchangeName(),
deliveredWithdrawInfoWithExchanges.second.receivedAmount());
break;
}
case CoincenterCommandType::kDustSweeper: {
- _queryResultPrinter.printDustSweeper(dustSweeper(cmd.exchangeNames(), cmd.cur1()), cmd.cur1());
+ _queryResultPrinter.printDustSweeper(dustSweeper(firstCmd.exchangeNames(), firstCmd.cur1()), firstCmd.cur1());
+ break;
+ }
+ case CoincenterCommandType::kMarketData: {
+ std::array marketPerPublicExchange;
+ for (const auto &cmd : groupedCommands) {
+ if (cmd.exchangeNames().empty()) {
+ std::ranges::fill(marketPerPublicExchange, cmd.market());
+ } else {
+ for (const auto &exchangeName : cmd.exchangeNames()) {
+ marketPerPublicExchange[exchangeName.publicExchangePos()] = cmd.market();
+ }
+ }
+ }
+ // No return value here, this command is made only for storing purposes.
+ queryMarketDataPerExchange(marketPerPublicExchange);
+ break;
+ }
+ case CoincenterCommandType::kReplay: {
+ /// This implementation of AbstractMarketTraderFactory is only provided as an example.
+ /// You can extend coincenter library and:
+ /// - Provide your own algorithms by implementing your own MarketTraderFactory will all your algorithms.
+ /// - Create your own CommandType that will call coincenter.replay with the same parameters as below, with your
+ /// own MarketTraderFactory.
+ MarketTraderFactory marketTraderFactory;
+ replay(marketTraderFactory, firstCmd.replayOptions(), firstCmd.market(), firstCmd.exchangeNames());
+ break;
+ }
+ case CoincenterCommandType::kReplayMarkets: {
+ const auto marketTimestampSetsPerExchange =
+ getMarketsAvailableForReplay(firstCmd.replayOptions(), firstCmd.exchangeNames());
+ _queryResultPrinter.printMarketsForReplay(firstCmd.replayOptions().timeWindow(), marketTimestampSetsPerExchange);
break;
}
default:
@@ -327,6 +379,40 @@ MarketOrderBookConversionRates Coincenter::getMarketOrderBooks(Market mk, Exchan
return ret;
}
+void Coincenter::queryMarketDataPerExchange(std::span marketPerPublicExchange) {
+ ExchangeNames exchangeNames;
+
+ int exchangePos{};
+ for (Market market : marketPerPublicExchange) {
+ if (market.isDefined()) {
+ exchangeNames.emplace_back(kSupportedExchanges[exchangePos]);
+ }
+ ++exchangePos;
+ }
+
+ const auto marketDataPerExchange =
+ _exchangesOrchestrator.getMarketDataPerExchange(marketPerPublicExchange, exchangeNames);
+
+ // Transform data structures to export metrics input format
+ MarketOrderBookConversionRates marketOrderBookConversionRates(marketDataPerExchange.size());
+ TradesPerExchange lastTradesPerExchange(marketDataPerExchange.size());
+
+ std::ranges::transform(marketDataPerExchange, marketOrderBookConversionRates.begin(),
+ [](const auto &exchangeWithPairOrderBooksAndTrades) {
+ return std::make_tuple(exchangeWithPairOrderBooksAndTrades.first->name(),
+ exchangeWithPairOrderBooksAndTrades.second.first, std::nullopt);
+ });
+
+ std::ranges::transform(marketDataPerExchange, lastTradesPerExchange.begin(),
+ [](const auto &exchangeWithPairOrderBooksAndTrades) {
+ return std::make_pair(exchangeWithPairOrderBooksAndTrades.first,
+ exchangeWithPairOrderBooksAndTrades.second.second);
+ });
+
+ _metricsExporter.exportOrderbookMetrics(marketOrderBookConversionRates);
+ _metricsExporter.exportLastTradesMetrics(lastTradesPerExchange);
+}
+
BalancePerExchange Coincenter::getBalance(std::span privateExchangeNames,
const BalanceOptions &balanceOptions) {
CurrencyCode equiCurrency = balanceOptions.equiCurrency();
@@ -459,6 +545,202 @@ MonetaryAmountPerExchange Coincenter::getLastPricePerExchange(Market mk, Exchang
return _exchangesOrchestrator.getLastPricePerExchange(mk, exchangeNames);
}
+MarketTimestampSetsPerExchange Coincenter::getMarketsAvailableForReplay(const ReplayOptions &replayOptions,
+ ExchangeNameSpan exchangeNames) {
+ return _exchangesOrchestrator.pullAvailableMarketsForReplay(replayOptions.timeWindow(), exchangeNames);
+}
+
+namespace {
+auto CreateExchangeNameVector(Market market, const MarketTimestampSetsPerExchange &marketTimestampSetsPerExchange) {
+ PublicExchangeNameVector exchangesWithThisMarketData;
+ for (const auto &[exchange, marketTimestampSets] : marketTimestampSetsPerExchange) {
+ if (ContainsMarket(market, marketTimestampSets)) {
+ exchangesWithThisMarketData.emplace_back(exchange->name());
+ }
+ }
+ return exchangesWithThisMarketData;
+}
+
+void CreateAndRegisterTraderAlgorithms(const AbstractMarketTraderFactory &marketTraderFactory,
+ std::string_view algorithmName,
+ std::span marketTraderEngines) {
+ for (auto &marketTraderEngine : marketTraderEngines) {
+ const auto &marketTraderEngineState = marketTraderEngine.marketTraderEngineState();
+
+ marketTraderEngine.registerMarketTrader(marketTraderFactory.construct(algorithmName, marketTraderEngineState));
+ }
+}
+
+bool Filter(Market market, MarketTimestampSet &marketTimestampSet) {
+ auto it = std::partition_point(marketTimestampSet.begin(), marketTimestampSet.end(),
+ [market](const auto &marketTimestamp) { return marketTimestamp.market < market; });
+ if (it != marketTimestampSet.end() && it->market == market) {
+ auto marketTimestamp = *it;
+ marketTimestampSet.clear();
+ marketTimestampSet.insert(marketTimestamp);
+ return false;
+ }
+
+ marketTimestampSet.clear();
+ return true;
+}
+
+void Filter(Market market, MarketTimestampSetsPerExchange &marketTimestampSetsPerExchange) {
+ for (auto it = marketTimestampSetsPerExchange.begin(); it != marketTimestampSetsPerExchange.end();) {
+ const bool orderBooksEmpty = Filter(market, it->second.orderBooksMarkets);
+ const bool tradesEmpty = Filter(market, it->second.tradesMarkets);
+
+ if (orderBooksEmpty && tradesEmpty) {
+ // no more data, remove the exchange entry completely
+ it = marketTimestampSetsPerExchange.erase(it);
+ } else {
+ ++it;
+ }
+ }
+}
+
+} // namespace
+
+void Coincenter::replay(const AbstractMarketTraderFactory &marketTraderFactory, const ReplayOptions &replayOptions,
+ Market market, ExchangeNameSpan exchangeNames) {
+ const TimeWindow timeWindow = replayOptions.timeWindow();
+ auto marketTimestampSetsPerExchange = _exchangesOrchestrator.pullAvailableMarketsForReplay(timeWindow, exchangeNames);
+
+ if (market.isDefined()) {
+ Filter(market, marketTimestampSetsPerExchange);
+ }
+
+ MarketSet allMarkets = ComputeAllMarkets(marketTimestampSetsPerExchange);
+
+ ReplayAlgorithmNameIterator replayAlgorithmNameIterator(replayOptions.algorithmNames(),
+ marketTraderFactory.allSupportedAlgorithms());
+
+ while (replayAlgorithmNameIterator.hasNext()) {
+ std::string_view algorithmName = replayAlgorithmNameIterator.next();
+
+ for (const Market replayMarket : allMarkets) {
+ auto exchangesWithThisMarketData = CreateExchangeNameVector(replayMarket, marketTimestampSetsPerExchange);
+
+ // Create the MarketTraderEngines based on this market, filtering out exchanges without available amount to
+ // trade
+ MarketTraderEngineVector marketTraderEngines =
+ createMarketTraderEngines(replayOptions, replayMarket, exchangesWithThisMarketData);
+
+ replayAlgorithm(marketTraderFactory, algorithmName, replayOptions, marketTraderEngines,
+ exchangesWithThisMarketData);
+ }
+ }
+}
+
+void Coincenter::replayAlgorithm(const AbstractMarketTraderFactory &marketTraderFactory, std::string_view algorithmName,
+ const ReplayOptions &replayOptions, std::span marketTraderEngines,
+ const PublicExchangeNameVector &exchangesWithThisMarketData) {
+ CreateAndRegisterTraderAlgorithms(marketTraderFactory, algorithmName, marketTraderEngines);
+
+ MarketTradeRangeStatsPerExchange tradeRangeStatsPerExchange =
+ tradingProcess(replayOptions, marketTraderEngines, exchangesWithThisMarketData);
+
+ // Finally retrieve and print results for this market
+ MarketTradingGlobalResultPerExchange marketTradingResultPerExchange =
+ _exchangesOrchestrator.getMarketTraderResultPerExchange(
+ marketTraderEngines, std::move(tradeRangeStatsPerExchange), exchangesWithThisMarketData);
+
+ _queryResultPrinter.printMarketTradingResults(replayOptions.timeWindow(), marketTradingResultPerExchange,
+ CoincenterCommandType::kReplay);
+}
+
+namespace {
+MonetaryAmount ComputeStartAmount(CurrencyCode currencyCode, MonetaryAmount convertedAmount) {
+ MonetaryAmount startAmount = convertedAmount;
+
+ if (startAmount.currencyCode() != currencyCode) {
+ // This is possible as conversion may use equivalent fiats and stable coins
+ log::info("Target converted currency is different from market one, replace with market currency {} -> {}",
+ startAmount.currencyCode(), currencyCode);
+ startAmount = MonetaryAmount(startAmount.amount(), currencyCode, startAmount.nbDecimals());
+ }
+
+ return startAmount;
+}
+} // namespace
+
+Coincenter::MarketTraderEngineVector Coincenter::createMarketTraderEngines(
+ const ReplayOptions &replayOptions, Market market, PublicExchangeNameVector &exchangesWithThisMarketData) {
+ auto nbExchanges = exchangesWithThisMarketData.size();
+
+ const auto &automationConfig = _coincenterInfo.generalConfig().tradingConfig().automationConfig();
+ const auto startBaseAmountEquivalent = automationConfig.startBaseAmountEquivalent();
+ const auto startQuoteAmountEquivalent = automationConfig.startQuoteAmountEquivalent();
+ const bool isValidateOnly = replayOptions.replayMode() == ReplayOptions::ReplayMode::kValidateOnly;
+
+ auto convertedBaseAmountPerExchange =
+ isValidateOnly ? MonetaryAmountPerExchange{}
+ : getConversion(startBaseAmountEquivalent, market.base(), exchangesWithThisMarketData);
+ auto convertedQuoteAmountPerExchange =
+ isValidateOnly ? MonetaryAmountPerExchange{}
+ : getConversion(startQuoteAmountEquivalent, market.quote(), exchangesWithThisMarketData);
+
+ MarketTraderEngineVector marketTraderEngines;
+ for (decltype(nbExchanges) exchangePos = 0; exchangePos < nbExchanges; ++exchangePos) {
+ const MonetaryAmount startBaseAmount =
+ isValidateOnly ? MonetaryAmount{0, market.base()}
+ : ComputeStartAmount(market.base(), convertedBaseAmountPerExchange[exchangePos].second);
+ const MonetaryAmount startQuoteAmount =
+ isValidateOnly ? MonetaryAmount{0, market.quote()}
+ : ComputeStartAmount(market.quote(), convertedQuoteAmountPerExchange[exchangePos].second);
+
+ if (!isValidateOnly && (startBaseAmount == 0 || startQuoteAmount == 0)) {
+ log::warn("Cannot convert to start base / quote amounts for {} ({} / {})",
+ exchangesWithThisMarketData[exchangePos], startBaseAmount, startQuoteAmount);
+ exchangesWithThisMarketData.erase(exchangesWithThisMarketData.begin() + exchangePos);
+ convertedBaseAmountPerExchange.erase(convertedBaseAmountPerExchange.begin() + exchangePos);
+ convertedQuoteAmountPerExchange.erase(convertedQuoteAmountPerExchange.begin() + exchangePos);
+ --exchangePos;
+ --nbExchanges;
+ continue;
+ }
+
+ const ExchangeConfig &exchangeConfig =
+ _coincenterInfo.exchangeConfig(exchangesWithThisMarketData[exchangePos].name());
+
+ marketTraderEngines.emplace_back(exchangeConfig, market, startBaseAmount, startQuoteAmount);
+ }
+ return marketTraderEngines;
+}
+
+MarketTradeRangeStatsPerExchange Coincenter::tradingProcess(const ReplayOptions &replayOptions,
+ std::span marketTraderEngines,
+ ExchangeNameSpan exchangesWithThisMarketData) {
+ const auto &automationConfig = _coincenterInfo.generalConfig().tradingConfig().automationConfig();
+ const auto loadChunkDuration = automationConfig.loadChunkDuration();
+ const auto timeWindow = replayOptions.timeWindow();
+
+ MarketTradeRangeStatsPerExchange tradeRangeResultsPerExchange;
+
+ // Main loop - parallelized by exchange, with time window chunks of loadChunkDuration
+
+ TimeWindow subTimeWindow(timeWindow.from(), loadChunkDuration);
+ while (subTimeWindow.overlaps(timeWindow)) {
+ auto subRangeResultsPerExchange = _exchangesOrchestrator.traderConsumeRange(
+ replayOptions, subTimeWindow, marketTraderEngines, exchangesWithThisMarketData);
+
+ if (tradeRangeResultsPerExchange.empty()) {
+ tradeRangeResultsPerExchange = std::move(subRangeResultsPerExchange);
+ } else {
+ int pos{};
+ for (auto &[exchange, result] : subRangeResultsPerExchange) {
+ tradeRangeResultsPerExchange[pos].second += result;
+ ++pos;
+ }
+ }
+
+ // Go to next sub time window
+ subTimeWindow = TimeWindow(subTimeWindow.to(), loadChunkDuration);
+ }
+
+ return tradeRangeResultsPerExchange;
+}
+
void Coincenter::updateFileCaches() const {
log::debug("Store all cache files");
diff --git a/src/engine/src/coincentercommand.cpp b/src/engine/src/coincentercommand.cpp
index a5bab733..b578dc93 100644
--- a/src/engine/src/coincentercommand.cpp
+++ b/src/engine/src/coincentercommand.cpp
@@ -11,6 +11,7 @@
#include "market.hpp"
#include "monetaryamount.hpp"
#include "ordersconstraints.hpp"
+#include "timedef.hpp"
#include "tradeoptions.hpp"
#include "withdrawoptions.hpp"
#include "withdrawsconstraints.hpp"
@@ -124,4 +125,10 @@ CoincenterCommand& CoincenterCommand::withBalanceInUse(bool value) {
_withBalanceInUse = value;
return *this;
}
+
+CoincenterCommand& CoincenterCommand::setReplayOptions(ReplayOptions replayOptions) {
+ _specialOptions = std::move(replayOptions);
+ return *this;
+}
+
} // namespace cct
diff --git a/src/engine/src/coincentercommands.cpp b/src/engine/src/coincentercommands.cpp
index 3d4550aa..f819a993 100644
--- a/src/engine/src/coincentercommands.cpp
+++ b/src/engine/src/coincentercommands.cpp
@@ -11,7 +11,10 @@
#include "coincenteroptions.hpp"
#include "currencycode.hpp"
#include "depositsconstraints.hpp"
+#include "market.hpp"
+#include "replay-options.hpp"
#include "stringoptionparser.hpp"
+#include "time-window.hpp"
#include "timedef.hpp"
#include "withdrawsconstraints.hpp"
@@ -199,6 +202,46 @@ void CoincenterCommands::addOption(const CoincenterCmdLineOptions &cmdLineOption
.setExchangeNames(optionParser.parseExchanges());
}
+ if (!cmdLineOptions.marketData.empty()) {
+ optionParser = StringOptionParser(cmdLineOptions.marketData);
+
+ _commands.emplace_back(CoincenterCommandType::kMarketData)
+ .setMarket(optionParser.parseMarket())
+ .setExchangeNames(optionParser.parseExchanges());
+ }
+
+ if (cmdLineOptions.replay) {
+ optionParser = StringOptionParser(*cmdLineOptions.replay);
+
+ auto dur = optionParser.parseDuration(StringOptionParser::FieldIs::kOptional);
+
+ auto &cmd = _commands.emplace_back(CoincenterCommandType::kReplay)
+ .setReplayOptions(cmdLineOptions.computeReplayOptions(dur))
+ .setExchangeNames(optionParser.parseExchanges());
+
+ if (!cmdLineOptions.market.empty()) {
+ cmd.setMarket(Market(cmdLineOptions.market));
+ }
+ }
+
+ if (cmdLineOptions.replayMarkets) {
+ optionParser = StringOptionParser(*cmdLineOptions.replayMarkets);
+
+ TimeWindow timeWindow;
+ auto dur = optionParser.parseDuration(StringOptionParser::FieldIs::kOptional);
+ auto nowTime = Clock::now();
+ if (dur == kUndefinedDuration) {
+ timeWindow = TimeWindow(TimePoint{}, nowTime);
+ } else {
+ timeWindow = TimeWindow(nowTime - dur, nowTime);
+ }
+
+ _commands.emplace_back(CoincenterCommandType::kReplayMarkets)
+ .setReplayOptions(
+ ReplayOptions(timeWindow, cmdLineOptions.algorithmNames, ReplayOptions::ReplayMode::kValidateOnly))
+ .setExchangeNames(optionParser.parseExchanges());
+ }
+
optionParser.checkEndParsing(); // No more option part should be remaining
}
diff --git a/src/engine/src/coincenterinfo_create.cpp b/src/engine/src/coincenterinfo_create.cpp
index a15663ec..225f734b 100644
--- a/src/engine/src/coincenterinfo_create.cpp
+++ b/src/engine/src/coincenterinfo_create.cpp
@@ -4,6 +4,7 @@
#include
#include "apioutputtype.hpp"
+#include "automation-config.hpp"
#include "cct_json.hpp"
#include "cct_string.hpp"
#include "coincenterinfo.hpp"
@@ -19,6 +20,7 @@
#include "runmodes.hpp"
#include "stringoptionparser.hpp"
#include "timedef.hpp"
+#include "trading-config.hpp"
namespace cct {
@@ -70,8 +72,21 @@ CoincenterInfo CoincenterInfo_Create(std::string_view programName, const Coincen
RequestsConfig requestsConfig(
generalConfigData.at("requests").at("concurrency").at("nbMaxParallelRequests").get());
- GeneralConfig generalConfig(std::move(loggingInfo), std::move(requestsConfig), fiatConversionQueryRate,
- apiOutputType);
+ const auto &automationJsonPart = generalConfigData.at("trading").at("automation");
+ const auto &deserializationJsonPart = automationJsonPart.at("deserialization");
+ const auto &startingContextJsonPart = automationJsonPart.at("startingContext");
+
+ Duration loadChunkDuration = ParseDuration(deserializationJsonPart.at("loadChunkDuration").get());
+ MonetaryAmount startBaseAmountEquivalent{
+ startingContextJsonPart.at("startBaseAmountEquivalent").get()};
+ MonetaryAmount startQuoteAmountEquivalent{
+ startingContextJsonPart.at("startQuoteAmountEquivalent").get()};
+
+ AutomationConfig automationConfig(loadChunkDuration, startBaseAmountEquivalent, startQuoteAmountEquivalent);
+ TradingConfig tradingConfig(std::move(automationConfig));
+
+ GeneralConfig generalConfig(std::move(loggingInfo), std::move(requestsConfig), std::move(tradingConfig),
+ fiatConversionQueryRate, apiOutputType);
const LoadConfiguration loadConfiguration(dataDir, LoadConfiguration::ExchangeConfigFileType::kProd);
diff --git a/src/engine/src/coincenteroptions.cpp b/src/engine/src/coincenteroptions.cpp
index 94f532ec..96488c43 100644
--- a/src/engine/src/coincenteroptions.cpp
+++ b/src/engine/src/coincenteroptions.cpp
@@ -13,7 +13,10 @@
#include "monetaryamount.hpp"
#include "priceoptions.hpp"
#include "priceoptionsdef.hpp"
+#include "replay-options.hpp"
#include "ssl_sha.hpp"
+#include "time-window.hpp"
+#include "timedef.hpp"
#include "tradedefinitions.hpp"
#include "tradeoptions.hpp"
#include "withdrawoptions.hpp"
@@ -37,6 +40,9 @@ std::ostream& CoincenterCmdLineOptions::PrintVersion(std::string_view programNam
os << "compiled with " << CCT_COMPILER_VERSION << " on " << __DATE__ << " at " << __TIME__ << '\n';
os << " " << GetCurlVersionInfo() << '\n';
os << " " << ssl::GetOpenSSLVersion() << '\n';
+#ifdef CCT_PROTOBUF_VERSION
+ os << " " << "protobuf " << CCT_PROTOBUF_VERSION << '\n';
+#endif
return os;
}
@@ -132,6 +138,31 @@ WithdrawOptions CoincenterCmdLineOptions::computeWithdrawOptions() const {
return {withdrawRefreshTime, withdrawSyncPolicy, mode};
}
+ReplayOptions CoincenterCmdLineOptions::computeReplayOptions(Duration dur) const {
+ if (validate && validateOnly) {
+ throw invalid_argument("--validate and --validate-only cannot be specified simultaneously");
+ }
+
+ ReplayOptions::ReplayMode replayMode;
+ if (validateOnly) {
+ replayMode = ReplayOptions::ReplayMode::kValidateOnly;
+ } else if (validate) {
+ replayMode = ReplayOptions::ReplayMode::kCheckedLaunchAlgorithm;
+ } else {
+ replayMode = ReplayOptions::ReplayMode::kUncheckedLaunchAlgorithm;
+ }
+
+ TimeWindow timeWindow;
+ const auto nowTime = Clock::now();
+ if (dur == kUndefinedDuration) {
+ timeWindow = TimeWindow(TimePoint{}, nowTime);
+ } else {
+ timeWindow = TimeWindow(nowTime - dur, nowTime);
+ }
+
+ return ReplayOptions(timeWindow, algorithmNames, replayMode);
+}
+
std::pair CoincenterCmdLineOptions::getTradeArgStr() const {
if (!tradeStrategy.empty() && !tradePrice.empty()) {
throw invalid_argument("Trade price and trade strategy cannot be set together");
diff --git a/src/engine/src/exchangesorchestrator.cpp b/src/engine/src/exchangesorchestrator.cpp
index 7132175f..b1347c74 100644
--- a/src/engine/src/exchangesorchestrator.cpp
+++ b/src/engine/src/exchangesorchestrator.cpp
@@ -31,13 +31,16 @@
#include "exchangepublicapi.hpp"
#include "exchangepublicapitypes.hpp"
#include "exchangeretriever.hpp"
+#include "market-trader-engine.hpp"
#include "market.hpp"
#include "monetaryamount.hpp"
#include "monetaryamountbycurrencyset.hpp"
#include "ordersconstraints.hpp"
#include "queryresulttypes.hpp"
+#include "replay-options.hpp"
#include "requestsconfig.hpp"
#include "threadpool.hpp"
+#include "trade-range-stats.hpp"
#include "tradedamounts.hpp"
#include "tradeoptions.hpp"
#include "traderesult.hpp"
@@ -170,7 +173,7 @@ MarketOrderBookConversionRates ExchangesOrchestrator::getMarketOrderBooks(Market
if (!optConversionRate && !equiCurrencyCode.isNeutral()) {
log::warn("Unable to convert {} into {} on {}", mk.quote(), equiCurrencyCode, exchange->name());
}
- return std::make_tuple(exchange->name(), exchange->queryOrderBook(mk, actualDepth), optConversionRate);
+ return std::make_tuple(exchange->name(), exchange->getOrderBook(mk, actualDepth), optConversionRate);
};
_threadPool.parallelTransform(selectedExchanges.begin(), selectedExchanges.end(), ret.begin(), marketOrderBooksFunc);
return ret;
@@ -941,7 +944,7 @@ TradesPerExchange ExchangesOrchestrator::getLastTradesPerExchange(Market mk, Exc
TradesPerExchange ret(selectedExchanges.size());
_threadPool.parallelTransform(
selectedExchanges.begin(), selectedExchanges.end(), ret.begin(), [mk, nbLastTrades](Exchange *exchange) {
- return std::make_pair(static_cast(exchange), exchange->queryLastTrades(mk, nbLastTrades));
+ return std::make_pair(static_cast(exchange), exchange->getLastTrades(mk, nbLastTrades));
});
return ret;
@@ -958,4 +961,116 @@ MonetaryAmountPerExchange ExchangesOrchestrator::getLastPricePerExchange(Market
return lastPricePerExchange;
}
+MarketDataPerExchange ExchangesOrchestrator::getMarketDataPerExchange(std::span marketPerPublicExchange,
+ ExchangeNameSpan exchangeNames) {
+ UniquePublicSelectedExchanges selectedExchanges = _exchangeRetriever.selectOneAccount(exchangeNames);
+
+ std::array isMarketTradable;
+
+ _threadPool.parallelTransform(selectedExchanges.begin(), selectedExchanges.end(), isMarketTradable.begin(),
+ [&marketPerPublicExchange](Exchange *exchange) {
+ Market market = marketPerPublicExchange[exchange->publicExchangePos()];
+ return market.isDefined() && exchange->queryTradableMarkets().contains(market);
+ });
+
+ FilterVector(selectedExchanges, isMarketTradable);
+
+ MarketDataPerExchange ret(selectedExchanges.size());
+ _threadPool.parallelTransform(
+ selectedExchanges.begin(), selectedExchanges.end(), ret.begin(), [&marketPerPublicExchange](Exchange *exchange) {
+ if (!exchange->exchangeConfig().withMarketDataSerialization()) {
+ log::warn("Calling market-data on {} with data serialization disabled", exchange->name());
+ }
+ // Call order book and last trades sequentially for this exchange
+ Market market = marketPerPublicExchange[exchange->publicExchangePos()];
+ return std::make_pair(exchange,
+ std::make_pair(exchange->getOrderBook(market), exchange->getLastTrades(market)));
+ });
+ return ret;
+}
+
+MarketTimestampSetsPerExchange ExchangesOrchestrator::pullAvailableMarketsForReplay(TimeWindow timeWindow,
+ ExchangeNameSpan exchangeNames) {
+ log::info("Query available markets for replay from {} within {}", ConstructAccumulatedExchangeNames(exchangeNames),
+ timeWindow);
+ UniquePublicSelectedExchanges selectedExchanges = _exchangeRetriever.selectOneAccount(exchangeNames);
+ MarketTimestampSetsPerExchange marketTimestampSetsPerExchange(selectedExchanges.size());
+ _threadPool.parallelTransform(selectedExchanges.begin(), selectedExchanges.end(),
+ marketTimestampSetsPerExchange.begin(), [timeWindow](Exchange *exchange) {
+ return std::make_pair(
+ exchange,
+ MarketTimestampSets{exchange->apiPublic().pullMarketOrderBooksMarkets(timeWindow),
+ exchange->apiPublic().pullTradeMarkets(timeWindow)});
+ });
+ return marketTimestampSetsPerExchange;
+}
+
+MarketTradeRangeStatsPerExchange ExchangesOrchestrator::traderConsumeRange(
+ const ReplayOptions &replayOptions, TimeWindow subTimeWindow, std::span marketTraderEngines,
+ ExchangeNameSpan exchangeNames) {
+ UniquePublicSelectedExchanges selectedExchanges = _exchangeRetriever.selectOneAccount(exchangeNames);
+
+ MarketTradeRangeStatsPerExchange tradeRangeResultsPerExchange(selectedExchanges.size());
+
+ _threadPool.parallelTransform(
+ selectedExchanges.begin(), selectedExchanges.end(), marketTraderEngines.begin(),
+ tradeRangeResultsPerExchange.begin(),
+ [subTimeWindow, &replayOptions](Exchange *exchange, MarketTraderEngine &marketTraderEngine) {
+ Market market = marketTraderEngine.market();
+ auto &apiPublic = exchange->apiPublic();
+ auto marketOrderBooks = apiPublic.pullMarketOrderBooksForReplay(market, subTimeWindow);
+ auto publicTrades = apiPublic.pullTradesForReplay(market, subTimeWindow);
+
+ TradeRangeStats tradeRangeStats;
+
+ switch (replayOptions.replayMode()) {
+ case ReplayOptions::ReplayMode::kValidateOnly:
+ tradeRangeStats = marketTraderEngine.validateRange(std::move(marketOrderBooks), std::move(publicTrades));
+ break;
+ case ReplayOptions::ReplayMode::kCheckedLaunchAlgorithm:
+ tradeRangeStats = marketTraderEngine.validateRange(marketOrderBooks, publicTrades);
+ marketTraderEngine.tradeRange(std::move(marketOrderBooks), std::move(publicTrades));
+ break;
+ case ReplayOptions::ReplayMode::kUncheckedLaunchAlgorithm:
+ tradeRangeStats = marketTraderEngine.tradeRange(std::move(marketOrderBooks), std::move(publicTrades));
+ break;
+ default:
+ break;
+ }
+
+ return std::make_pair(exchange, std::move(tradeRangeStats));
+ });
+
+ return tradeRangeResultsPerExchange;
+}
+
+MarketTradingGlobalResultPerExchange ExchangesOrchestrator::getMarketTraderResultPerExchange(
+ std::span marketTraderEngines, MarketTradeRangeStatsPerExchange &&tradeRangeStatsPerExchange,
+ ExchangeNameSpan exchangeNames) {
+ UniquePublicSelectedExchanges selectedExchanges = _exchangeRetriever.selectOneAccount(exchangeNames);
+
+ if (selectedExchanges.size() != tradeRangeStatsPerExchange.size()) {
+ throw exception("Inconsistent selected exchange sizes");
+ }
+
+ MarketTradingResultPerExchange marketTradingResultPerExchange(selectedExchanges.size());
+
+ _threadPool.parallelTransform(selectedExchanges.begin(), selectedExchanges.end(), marketTraderEngines.begin(),
+ marketTradingResultPerExchange.begin(),
+ [](const Exchange *exchange, MarketTraderEngine &marketTraderEngine) {
+ return std::make_pair(exchange, marketTraderEngine.finalizeAndComputeResult());
+ });
+
+ MarketTradingGlobalResultPerExchange marketTradingGlobalResultPerExchange(selectedExchanges.size());
+ std::transform(marketTradingResultPerExchange.begin(), marketTradingResultPerExchange.end(),
+ tradeRangeStatsPerExchange.begin(), marketTradingGlobalResultPerExchange.begin(),
+ [](auto &exchangeMarketTradingResult, auto &exchangeTradeRangeStats) {
+ return std::make_pair(exchangeMarketTradingResult.first,
+ MarketTradingGlobalResult{std::move(exchangeMarketTradingResult.second),
+ std::move(exchangeTradeRangeStats.second)});
+ });
+
+ return marketTradingGlobalResultPerExchange;
+}
+
} // namespace cct
diff --git a/src/engine/src/query-result-type-helpers.cpp b/src/engine/src/query-result-type-helpers.cpp
new file mode 100644
index 00000000..866aec2f
--- /dev/null
+++ b/src/engine/src/query-result-type-helpers.cpp
@@ -0,0 +1,34 @@
+#include "query-result-type-helpers.hpp"
+
+#include
+#include
+
+#include "exchangepublicapitypes.hpp"
+#include "market-timestamp-set.hpp"
+#include "market.hpp"
+#include "queryresulttypes.hpp"
+
+namespace cct {
+
+bool ContainsMarket(Market market, const MarketTimestampSet &marketTimestampSet) {
+ auto it = std::ranges::partition_point(
+ marketTimestampSet, [market](const auto &marketTimestamp) { return marketTimestamp.market < market; });
+ return it != marketTimestampSet.end() && it->market == market;
+}
+
+bool ContainsMarket(Market market, const MarketTimestampSets &marketTimestampSets) {
+ return ContainsMarket(market, marketTimestampSets.orderBooksMarkets) ||
+ ContainsMarket(market, marketTimestampSets.tradesMarkets);
+}
+
+MarketSet ComputeAllMarkets(const MarketTimestampSetsPerExchange &marketTimestampSetsPerExchange) {
+ MarketSet allMarkets;
+ for (const auto &[_, marketTimestamps] : marketTimestampSetsPerExchange) {
+ std::ranges::transform(marketTimestamps.orderBooksMarkets, std::inserter(allMarkets, allMarkets.end()),
+ [](const auto &marketTimestamp) { return marketTimestamp.market; });
+ std::ranges::transform(marketTimestamps.tradesMarkets, std::inserter(allMarkets, allMarkets.end()),
+ [](const auto &marketTimestamp) { return marketTimestamp.market; });
+ }
+ return allMarkets;
+}
+} // namespace cct
\ No newline at end of file
diff --git a/src/engine/src/queryresultprinter.cpp b/src/engine/src/queryresultprinter.cpp
index 67bd223b..678ff1d2 100644
--- a/src/engine/src/queryresultprinter.cpp
+++ b/src/engine/src/queryresultprinter.cpp
@@ -27,6 +27,7 @@
#include "exchange.hpp"
#include "file.hpp"
#include "logginginfo.hpp"
+#include "market-timestamp.hpp"
#include "market.hpp"
#include "marketorderbook.hpp"
#include "monetaryamount.hpp"
@@ -36,9 +37,11 @@
#include "priceoptions.hpp"
#include "priceoptionsdef.hpp"
#include "publictrade.hpp"
+#include "query-result-type-helpers.hpp"
#include "queryresulttypes.hpp"
#include "simpletable.hpp"
#include "stringhelpers.hpp"
+#include "time-window.hpp"
#include "timestring.hpp"
#include "tradedamounts.hpp"
#include "tradedefinitions.hpp"
@@ -126,6 +129,47 @@ json MarketsJson(CurrencyCode cur1, CurrencyCode cur2, const MarketsPerExchange
return ToJson(CoincenterCommandType::kMarkets, std::move(in), std::move(out));
}
+json MarketsForReplayJson(TimeWindow timeWindow, const MarketTimestampSetsPerExchange &marketTimestampSetsPerExchange) {
+ json in;
+ json inOpt = json::object();
+ if (timeWindow != TimeWindow{}) {
+ inOpt.emplace("timeWindow", timeWindow.str());
+ }
+ in.emplace("opt", std::move(inOpt));
+
+ json out = json::object();
+ for (const auto &[e, marketTimestampSets] : marketTimestampSetsPerExchange) {
+ json orderBookMarketsPerExchange;
+ for (const MarketTimestamp &marketTimestamp : marketTimestampSets.orderBooksMarkets) {
+ json marketTimestampJson;
+
+ marketTimestampJson.emplace("market", marketTimestamp.market.str());
+ marketTimestampJson.emplace("lastTimestamp", ToString(marketTimestamp.timePoint));
+
+ orderBookMarketsPerExchange.emplace_back(std::move(marketTimestampJson));
+ }
+
+ json tradesMarketsPerExchange;
+ for (const MarketTimestamp &marketTimestamp : marketTimestampSets.tradesMarkets) {
+ json marketTimestampJson;
+
+ marketTimestampJson.emplace("market", marketTimestamp.market.str());
+ marketTimestampJson.emplace("lastTimestamp", ToString(marketTimestamp.timePoint));
+
+ tradesMarketsPerExchange.emplace_back(std::move(marketTimestampJson));
+ }
+
+ json exchangePart;
+
+ exchangePart.emplace("orderBooks", std::move(orderBookMarketsPerExchange));
+ exchangePart.emplace("trades", std::move(tradesMarketsPerExchange));
+
+ out.emplace(e->name(), std::move(exchangePart));
+ }
+
+ return ToJson(CoincenterCommandType::kReplayMarkets, std::move(in), std::move(out));
+}
+
json TickerInformationJson(const ExchangeTickerMaps &exchangeTickerMaps) {
json in;
json out = json::object();
@@ -722,6 +766,57 @@ json DustSweeperJson(const TradedAmountsVectorWithFinalAmountPerExchange &traded
return ToJson(CoincenterCommandType::kDustSweeper, std::move(in), std::move(out));
}
+json MarketTradingResultsJson(TimeWindow timeWindow,
+ const MarketTradingGlobalResultPerExchange &marketTradingResultPerExchange,
+ CoincenterCommandType commandType) {
+ json in;
+ json inOpt;
+ inOpt.emplace("time-window", timeWindow.str());
+ in.emplace("opt", std::move(inOpt));
+
+ json out = json::object();
+
+ for (const auto &[exchangePtr, marketGlobalTradingResult] : marketTradingResultPerExchange) {
+ const auto &marketTradingResult = marketGlobalTradingResult.result;
+ const auto &stats = marketGlobalTradingResult.stats;
+
+ json startAmounts;
+ startAmounts.emplace("base", marketTradingResult.startBaseAmount().str());
+ startAmounts.emplace("quote", marketTradingResult.startQuoteAmount().str());
+
+ json orderBookStats;
+ orderBookStats.emplace("nb-successful", stats.marketOrderBookStats.nbSuccessful);
+ orderBookStats.emplace("nb-error", stats.marketOrderBookStats.nbError);
+
+ json tradeStats;
+ tradeStats.emplace("nb-successful", stats.publicTradeStats.nbSuccessful);
+ tradeStats.emplace("nb-error", stats.publicTradeStats.nbError);
+
+ json jsonStats;
+ jsonStats.emplace("order-books", std::move(orderBookStats));
+ jsonStats.emplace("trades", std::move(tradeStats));
+
+ json marketTradingResultJson;
+ marketTradingResultJson.emplace("algorithm", marketTradingResult.algorithmName());
+ marketTradingResultJson.emplace("market", marketTradingResult.market().str());
+ marketTradingResultJson.emplace("start-amounts", std::move(startAmounts));
+ marketTradingResultJson.emplace("profit-and-loss", marketTradingResult.quoteAmountDelta().str());
+ marketTradingResultJson.emplace("stats", std::move(jsonStats));
+
+ json closedOrdersArray = json::array_t();
+
+ for (const ClosedOrder &closedOrder : marketTradingResult.matchedOrders()) {
+ closedOrdersArray.push_back(OrderJson(closedOrder));
+ }
+
+ marketTradingResultJson.emplace("matched-orders", std::move(closedOrdersArray));
+
+ out.emplace(exchangePtr->name(), std::move(marketTradingResultJson));
+ }
+
+ return ToJson(commandType, std::move(in), std::move(out));
+}
+
template
void RemoveDuplicates(VecType &vec) {
std::ranges::sort(vec);
@@ -1452,6 +1547,123 @@ void QueryResultPrinter::printDustSweeper(
logActivity(CoincenterCommandType::kDustSweeper, jsonData);
}
+void QueryResultPrinter::printMarketsForReplay(TimeWindow timeWindow,
+ const MarketTimestampSetsPerExchange &marketTimestampSetsPerExchange) {
+ json jsonData = MarketsForReplayJson(timeWindow, marketTimestampSetsPerExchange);
+ switch (_apiOutputType) {
+ case ApiOutputType::kFormattedTable: {
+ MarketSet allMarkets = ComputeAllMarkets(marketTimestampSetsPerExchange);
+
+ SimpleTable table;
+ table.reserve(allMarkets.size() + 1U);
+ table.emplace_back("Markets", "Last order books timestamp", "Last trades timestamp");
+
+ for (const Market market : allMarkets) {
+ table::Cell orderBookCell;
+ table::Cell tradesCell;
+ for (const auto &[e, marketTimestamps] : marketTimestampSetsPerExchange) {
+ const auto &orderBooksMarkets = marketTimestamps.orderBooksMarkets;
+ const auto &tradesMarkets = marketTimestamps.tradesMarkets;
+ const auto marketPartitionPred = [market](const auto &marketTimestamp) {
+ return marketTimestamp.market < market;
+ };
+ const auto orderBooksIt = std::ranges::partition_point(orderBooksMarkets, marketPartitionPred);
+ const auto tradesIt = std::ranges::partition_point(tradesMarkets, marketPartitionPred);
+
+ if (orderBooksIt != orderBooksMarkets.end() && orderBooksIt->market == market) {
+ string str = ToString(orderBooksIt->timePoint);
+ str.append(" @ ");
+ str.append(e->name());
+
+ orderBookCell.emplace_back(std::move(str));
+ }
+
+ if (tradesIt != tradesMarkets.end() && tradesIt->market == market) {
+ string str = ToString(tradesIt->timePoint);
+ str.append(" @ ");
+ str.append(e->name());
+
+ tradesCell.emplace_back(std::move(str));
+ }
+ }
+
+ table.emplace_back(market.str(), std::move(orderBookCell), std::move(tradesCell));
+ }
+ printTable(table);
+ break;
+ }
+ case ApiOutputType::kJson:
+ printJson(jsonData);
+ break;
+ case ApiOutputType::kNoPrint:
+ break;
+ }
+ logActivity(CoincenterCommandType::kReplayMarkets, jsonData);
+}
+
+void QueryResultPrinter::printMarketTradingResults(
+ TimeWindow timeWindow, const MarketTradingGlobalResultPerExchange &marketTradingResultPerExchange,
+ CoincenterCommandType commandType) const {
+ json jsonData = MarketTradingResultsJson(timeWindow, marketTradingResultPerExchange, commandType);
+ switch (_apiOutputType) {
+ case ApiOutputType::kFormattedTable: {
+ SimpleTable table;
+ table.reserve(1U + marketTradingResultPerExchange.size());
+ table.emplace_back("Exchange", "Time window", "Market", "Algorithm", "Start amounts", "Profit / Loss",
+ "Matched orders", "Stats");
+ for (const auto &[exchangePtr, marketGlobalTradingResults] : marketTradingResultPerExchange) {
+ const auto &marketTradingResults = marketGlobalTradingResults.result;
+ const auto &stats = marketGlobalTradingResults.stats;
+
+ table::Cell trades;
+ for (const ClosedOrder &closedOrder : marketTradingResults.matchedOrders()) {
+ string orderStr = closedOrder.placedTimeStr();
+ orderStr.append(" - ");
+ orderStr.append(closedOrder.sideStr());
+ orderStr.append(" - ");
+ orderStr.append(closedOrder.matchedVolume().str());
+ orderStr.append(" @ ");
+ orderStr.append(closedOrder.price().str());
+ trades.emplace_back(std::move(orderStr));
+ }
+
+ string orderBookStats("order books: ");
+ orderBookStats.append(std::string_view(ToCharVector(stats.marketOrderBookStats.nbSuccessful)));
+ orderBookStats.append(" OK");
+ if (stats.marketOrderBookStats.nbError != 0) {
+ orderBookStats.append(", ");
+ orderBookStats.append(std::string_view(ToCharVector(stats.marketOrderBookStats.nbError)));
+ orderBookStats.append(" KO");
+ }
+
+ string tradesStats("trades: ");
+ tradesStats.append(std::string_view(ToCharVector(stats.publicTradeStats.nbSuccessful)));
+ tradesStats.append(" OK");
+ if (stats.publicTradeStats.nbError != 0) {
+ tradesStats.append(", ");
+ tradesStats.append(std::string_view(ToCharVector(stats.publicTradeStats.nbError)));
+ tradesStats.append(" KO");
+ }
+
+ table.emplace_back(
+ exchangePtr->name(), table::Cell{ToString(timeWindow.from()), ToString(timeWindow.to())},
+ marketTradingResults.market().str(), marketTradingResults.algorithmName(),
+ table::Cell{marketTradingResults.startBaseAmount().str(), marketTradingResults.startQuoteAmount().str()},
+ marketTradingResults.quoteAmountDelta().str(), std::move(trades),
+ table::Cell{std::move(orderBookStats), std::move(tradesStats)});
+ }
+ printTable(table);
+ break;
+ }
+ case ApiOutputType::kJson:
+ printJson(jsonData);
+ break;
+ case ApiOutputType::kNoPrint:
+ break;
+ }
+ logActivity(commandType, jsonData);
+}
+
void QueryResultPrinter::printTable(const SimpleTable &table) const {
std::ostringstream ss;
std::ostream &os = _pOs != nullptr ? *_pOs : ss;
diff --git a/src/engine/src/replay-algorithm-name-iterator.cpp b/src/engine/src/replay-algorithm-name-iterator.cpp
new file mode 100644
index 00000000..1a35a3a8
--- /dev/null
+++ b/src/engine/src/replay-algorithm-name-iterator.cpp
@@ -0,0 +1,63 @@
+#include "replay-algorithm-name-iterator.hpp"
+
+#include
+#include
+#include
+
+#include "cct_exception.hpp"
+
+namespace cct {
+
+namespace {
+constexpr std::string_view kAlgorithmNameSeparator = ",";
+
+auto FindNextSeparatorPos(std::string_view str, std::string_view::size_type pos = 0) {
+ pos = str.find(kAlgorithmNameSeparator, pos);
+ if (pos == std::string_view::npos) {
+ pos = str.length();
+ }
+ return pos;
+}
+} // namespace
+
+ReplayAlgorithmNameIterator::ReplayAlgorithmNameIterator(std::string_view algorithmNames,
+ std::span allAlgorithms)
+ : _allAlgorithms(allAlgorithms),
+ _algorithmNames(algorithmNames),
+ _begPos(0),
+ _endPos(FindNextSeparatorPos(_algorithmNames)) {
+ if (std::ranges::any_of(allAlgorithms, [](const auto algName) {
+ return algName.find(kAlgorithmNameSeparator) != std::string_view::npos;
+ })) {
+ throw exception("Algorithm names cannot contain '{}' as it's used as a separator", kAlgorithmNameSeparator);
+ }
+}
+
+bool ReplayAlgorithmNameIterator::hasNext() const {
+ using PosT = decltype(_begPos);
+
+ if (_algorithmNames.empty()) {
+ return _begPos < static_cast(_allAlgorithms.size());
+ }
+
+ return _begPos != static_cast(_algorithmNames.length());
+}
+
+std::string_view ReplayAlgorithmNameIterator::next() {
+ if (_algorithmNames.empty()) {
+ return _allAlgorithms[_begPos++];
+ }
+
+ std::string_view nextAlgorithmName(_algorithmNames.begin() + _begPos, _algorithmNames.begin() + _endPos);
+
+ if (_endPos == static_cast(_algorithmNames.length())) {
+ _begPos = _endPos;
+ } else {
+ _begPos = _endPos + kAlgorithmNameSeparator.length();
+ _endPos = FindNextSeparatorPos(_algorithmNames, _begPos);
+ }
+
+ return nextAlgorithmName;
+}
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/engine/src/replay-options.cpp b/src/engine/src/replay-options.cpp
new file mode 100644
index 00000000..7eb728c2
--- /dev/null
+++ b/src/engine/src/replay-options.cpp
@@ -0,0 +1,20 @@
+#include "replay-options.hpp"
+
+#include
+
+#include "dummy-market-trader.hpp"
+#include "time-window.hpp"
+
+namespace cct {
+
+ReplayOptions::ReplayOptions(TimeWindow timeWindow, std::string_view algorithmNames, ReplayMode replayMode)
+ : _timeWindow(timeWindow), _algorithmNames(algorithmNames), _replayMode(replayMode) {}
+
+std::string_view ReplayOptions::algorithmNames() const {
+ if (_replayMode == ReplayMode::kValidateOnly) {
+ return DummyMarketTrader::kName;
+ }
+ return _algorithmNames;
+}
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/engine/src/stringoptionparser.cpp b/src/engine/src/stringoptionparser.cpp
index 11502459..6435837d 100644
--- a/src/engine/src/stringoptionparser.cpp
+++ b/src/engine/src/stringoptionparser.cpp
@@ -9,10 +9,12 @@
#include "cct_string.hpp"
#include "cct_vector.hpp"
#include "currencycode.hpp"
+#include "durationstring.hpp"
#include "exchange-names.hpp"
#include "exchangename.hpp"
#include "market.hpp"
#include "monetaryamount.hpp"
+#include "timedef.hpp"
namespace cct {
@@ -38,6 +40,23 @@ CurrencyCode StringOptionParser::parseCurrency(FieldIs fieldIs, char delimiter)
return {};
}
+Duration StringOptionParser::parseDuration(FieldIs fieldIs) {
+ auto dur = kUndefinedDuration;
+ const std::string_view currentToken(_opt.begin() + _pos, _opt.end());
+ const auto durationLen = DurationLen(currentToken);
+ if (durationLen > 0) {
+ const std::string_view durationStr(_opt.data() + _pos, static_cast(durationLen));
+
+ dur = ParseDuration(durationStr);
+ } else if (fieldIs == FieldIs::kMandatory) {
+ throw invalid_argument("Expected a valid duration in '{}'", currentToken);
+ }
+
+ _pos += durationLen;
+
+ return dur;
+}
+
// At the end of the market, either the end of the string or a comma is expected.
Market StringOptionParser::parseMarket(FieldIs fieldIs, char delimiter) {
const auto oldPos = _pos;
diff --git a/src/engine/test/queryresultprinter_public_test.cpp b/src/engine/test/queryresultprinter_public_test.cpp
index 57bbc12e..6ba960df 100644
--- a/src/engine/test/queryresultprinter_public_test.cpp
+++ b/src/engine/test/queryresultprinter_public_test.cpp
@@ -9,15 +9,21 @@
#include "currencycode.hpp"
#include "currencyexchange.hpp"
#include "currencyexchangeflatset.hpp"
+#include "exchangeprivateapitypes.hpp"
#include "exchangepublicapitypes.hpp"
+#include "market-trading-global-result.hpp"
+#include "market-trading-result.hpp"
#include "market.hpp"
#include "marketorderbook.hpp"
#include "monetaryamount.hpp"
#include "monetaryamountbycurrencyset.hpp"
+#include "public-trade-vector.hpp"
#include "publictrade.hpp"
#include "queryresultprinter.hpp"
#include "queryresultprinter_base_test.hpp"
#include "queryresulttypes.hpp"
+#include "time-window.hpp"
+#include "trade-range-stats.hpp"
#include "tradeside.hpp"
namespace cct {
@@ -243,7 +249,6 @@ TEST_F(QueryResultPrinterMarketsTest, FormattedTableNoCurrency) {
| huobi | XRP-EUR |
+----------+---------+
)";
-
expectStr(kExpected);
}
@@ -262,7 +267,6 @@ TEST_F(QueryResultPrinterMarketsTest, FormattedTableOneCurrency) {
| huobi | XRP-EUR |
+----------+------------------+
)";
-
expectStr(kExpected);
}
@@ -279,7 +283,6 @@ TEST_F(QueryResultPrinterMarketsTest, FormattedTableTwoCurrencies) {
| huobi | XRP-EUR |
+----------+----------------------+
)";
-
expectStr(kExpected);
}
@@ -1176,4 +1179,303 @@ TEST_F(QueryResultPrinterLastPriceTest, NoPrint) {
expectNoStr();
}
+class QueryResultPrinterReplayBaseTest : public QueryResultPrinterTest {
+ protected:
+ Market market1{"ETH", "KRW"};
+ Market market2{"BTC", "USD"};
+ Market market3{"SHIB", "USDT"};
+ Market market4{"SOL", "BTC"};
+ Market market5{"SOL", "ETH"};
+ Market market6{"ETH", "BTC"};
+ Market market7{"DOGE", "CAD"};
+
+ TimePoint tp1{milliseconds{std::numeric_limits::max() / 10000000}};
+ TimePoint tp2{milliseconds{std::numeric_limits::max() / 9900000}};
+ TimePoint tp3{milliseconds{std::numeric_limits::max() / 9800000}};
+ TimePoint tp4{milliseconds{std::numeric_limits::max() / 9600000}};
+ TimePoint tp5{milliseconds{std::numeric_limits::max() / 9500000}};
+
+ TimeWindow timeWindow{tp1, tp5};
+};
+
+class QueryResultPrinterReplayMarketsTest : public QueryResultPrinterReplayBaseTest {
+ protected:
+ MarketTimestampSetsPerExchange marketTimestampSetsPerExchange{
+ {&exchange1,
+ MarketTimestampSets{MarketTimestampSet{MarketTimestamp{market1, tp1}, MarketTimestamp{market2, tp2},
+ MarketTimestamp{market3, tp3}},
+ MarketTimestampSet{MarketTimestamp{market1, tp1}, MarketTimestamp{market2, tp1}}}},
+ {&exchange2, MarketTimestampSets{MarketTimestampSet{MarketTimestamp{market2, tp4}, MarketTimestamp{market4, tp5}},
+ MarketTimestampSet{MarketTimestamp{market6, tp1}}}},
+ {&exchange3, MarketTimestampSets{MarketTimestampSet{}, MarketTimestampSet{MarketTimestamp{market1, tp1},
+ MarketTimestamp{market7, tp4}}}}};
+};
+
+TEST_F(QueryResultPrinterReplayMarketsTest, FormattedTable) {
+ basicQueryResultPrinter(ApiOutputType::kFormattedTable)
+ .printMarketsForReplay(timeWindow, marketTimestampSetsPerExchange);
+ static constexpr std::string_view kExpected = R"(
++-----------+--------------------------------+--------------------------------+
+| Markets | Last order books timestamp | Last trades timestamp |
++-----------+--------------------------------+--------------------------------+
+| BTC-USD | 1999-07-11T00:42:21Z @ binance | 1999-03-25T04:46:43Z @ binance |
+| | 2000-06-11T23:58:40Z @ bithumb | |
+|~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|
+| DOGE-CAD | | 2000-06-11T23:58:40Z @ huobi |
+| ETH-BTC | | 1999-03-25T04:46:43Z @ bithumb |
+|~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|
+| ETH-KRW | 1999-03-25T04:46:43Z @ binance | 1999-03-25T04:46:43Z @ binance |
+| | | 1999-03-25T04:46:43Z @ huobi |
+|~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|
+| SHIB-USDT | 1999-10-29T01:26:51Z @ binance | |
+| SOL-BTC | 2000-10-07T01:14:27Z @ bithumb | |
++-----------+--------------------------------+--------------------------------+
+)";
+ expectStr(kExpected);
+}
+
+TEST_F(QueryResultPrinterReplayMarketsTest, EmptyJson) {
+ basicQueryResultPrinter(ApiOutputType::kJson).printMarketsForReplay(timeWindow, MarketTimestampSetsPerExchange{});
+ static constexpr std::string_view kExpected = R"json(
+{
+ "in": {
+ "opt": {
+ "timeWindow": "[1999-03-25 04:46:43 -> 2000-10-07 01:14:27)"
+ },
+ "req": "ReplayMarkets"
+ },
+ "out": {}
+})json";
+ expectJson(kExpected);
+}
+
+TEST_F(QueryResultPrinterReplayMarketsTest, Json) {
+ basicQueryResultPrinter(ApiOutputType::kJson).printMarketsForReplay(timeWindow, marketTimestampSetsPerExchange);
+ static constexpr std::string_view kExpected = R"json(
+{
+ "in": {
+ "opt": {
+ "timeWindow": "[1999-03-25 04:46:43 -> 2000-10-07 01:14:27)"
+ },
+ "req": "ReplayMarkets"
+ },
+ "out": {
+ "binance": {
+ "orderBooks": [
+ {
+ "lastTimestamp": "1999-07-11T00:42:21Z",
+ "market": "BTC-USD"
+ },
+ {
+ "lastTimestamp": "1999-03-25T04:46:43Z",
+ "market": "ETH-KRW"
+ },
+ {
+ "lastTimestamp": "1999-10-29T01:26:51Z",
+ "market": "SHIB-USDT"
+ }
+ ],
+ "trades": [
+ {
+ "lastTimestamp": "1999-03-25T04:46:43Z",
+ "market": "BTC-USD"
+ },
+ {
+ "lastTimestamp": "1999-03-25T04:46:43Z",
+ "market": "ETH-KRW"
+ }
+ ]
+ },
+ "bithumb": {
+ "orderBooks": [
+ {
+ "lastTimestamp": "2000-06-11T23:58:40Z",
+ "market": "BTC-USD"
+ },
+ {
+ "lastTimestamp": "2000-10-07T01:14:27Z",
+ "market": "SOL-BTC"
+ }
+ ],
+ "trades": [
+ {
+ "lastTimestamp": "1999-03-25T04:46:43Z",
+ "market": "ETH-BTC"
+ }
+ ]
+ },
+ "huobi": {
+ "orderBooks": null,
+ "trades": [
+ {
+ "lastTimestamp": "2000-06-11T23:58:40Z",
+ "market": "DOGE-CAD"
+ },
+ {
+ "lastTimestamp": "1999-03-25T04:46:43Z",
+ "market": "ETH-KRW"
+ }
+ ]
+ }
+ }
+})json";
+ expectJson(kExpected);
+}
+
+TEST_F(QueryResultPrinterReplayMarketsTest, NoPrint) {
+ basicQueryResultPrinter(ApiOutputType::kNoPrint).printMarketsForReplay(timeWindow, marketTimestampSetsPerExchange);
+ expectNoStr();
+}
+
+class QueryResultPrinterReplayTest : public QueryResultPrinterReplayBaseTest {
+ protected:
+ ClosedOrder closedOrder1{"1", MonetaryAmount(15, "BTC", 1), MonetaryAmount(35000, "USDT"), tp1, tp1, TradeSide::kBuy};
+ ClosedOrder closedOrder2{"2", MonetaryAmount(25, "BTC", 1), MonetaryAmount(45000, "USDT"), tp2, tp2, TradeSide::kBuy};
+ ClosedOrder closedOrder3{"3", MonetaryAmount(5, "BTC", 2), MonetaryAmount(35000, "USDT"), tp3, tp4, TradeSide::kSell};
+ ClosedOrder closedOrder4{
+ "4", MonetaryAmount(17, "BTC", 1), MonetaryAmount(50000, "USDT"), tp3, tp4, TradeSide::kSell};
+ ClosedOrder closedOrder5{
+ "5", MonetaryAmount(36, "BTC", 3), MonetaryAmount(47899, "USDT"), tp4, tp5, TradeSide::kSell};
+
+ std::string_view algorithmName = "test-algo";
+ MonetaryAmount startBaseAmount{1, "BTC"};
+ MonetaryAmount startQuoteAmount{1000, "EUR"};
+
+ MarketTradingResult marketTradingResult1{algorithmName, startBaseAmount, startQuoteAmount, MonetaryAmount{0, "EUR"},
+ ClosedOrderVector{}};
+ MarketTradingResult marketTradingResult3{algorithmName, startBaseAmount, startQuoteAmount, MonetaryAmount{500, "EUR"},
+ ClosedOrderVector{closedOrder1, closedOrder5}};
+ MarketTradingResult marketTradingResult4{algorithmName, startBaseAmount, startQuoteAmount, MonetaryAmount{780, "EUR"},
+ ClosedOrderVector{closedOrder2, closedOrder3, closedOrder4}};
+
+ TradeRangeStats tradeRangeStats1{TradeRangeResultsStats{42, 0}, TradeRangeResultsStats{3, 10}};
+ TradeRangeStats tradeRangeStats3{TradeRangeResultsStats{500000, 2}, TradeRangeResultsStats{0, 0}};
+ TradeRangeStats tradeRangeStats4{TradeRangeResultsStats{79009, 0}, TradeRangeResultsStats{1555555555, 45}};
+
+ MarketTradingGlobalResultPerExchange marketTradingResultPerExchange{
+ {&exchange1, MarketTradingGlobalResult{marketTradingResult1, tradeRangeStats1}},
+ {&exchange3, MarketTradingGlobalResult{marketTradingResult3, tradeRangeStats3}},
+ {&exchange4, MarketTradingGlobalResult{marketTradingResult4, tradeRangeStats4}}};
+ CoincenterCommandType commandType{CoincenterCommandType::kReplay};
+};
+
+TEST_F(QueryResultPrinterReplayTest, FormattedTable) {
+ basicQueryResultPrinter(ApiOutputType::kFormattedTable)
+ .printMarketTradingResults(timeWindow, marketTradingResultPerExchange, commandType);
+ static constexpr std::string_view kExpected = R"(
++----------+----------------------+---------+-----------+---------------+---------------+------------------------------------------------------+------------------------------+
+| Exchange | Time window | Market | Algorithm | Start amounts | Profit / Loss | Matched orders | Stats |
++----------+----------------------+---------+-----------+---------------+---------------+------------------------------------------------------+------------------------------+
+| binance | 1999-03-25T04:46:43Z | BTC-EUR | test-algo | 1 BTC | 0 EUR | | order books: 42 OK |
+| | 2000-10-07T01:14:27Z | | | 1000 EUR | | | trades: 3 OK, 10 KO |
+|~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~|~~~~~~~~~|~~~~~~~~~~~|~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|
+| huobi | 1999-03-25T04:46:43Z | BTC-EUR | test-algo | 1 BTC | 500 EUR | 1999-03-25T04:46:43Z - Buy - 1.5 BTC @ 35000 USDT | order books: 500000 OK, 2 KO |
+| | 2000-10-07T01:14:27Z | | | 1000 EUR | | 2000-06-11T23:58:40Z - Sell - 0.036 BTC @ 47899 USDT | trades: 0 OK |
+|~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~|~~~~~~~~~|~~~~~~~~~~~|~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|
+| huobi | 1999-03-25T04:46:43Z | BTC-EUR | test-algo | 1 BTC | 780 EUR | 1999-07-11T00:42:21Z - Buy - 2.5 BTC @ 45000 USDT | order books: 79009 OK |
+| | 2000-10-07T01:14:27Z | | | 1000 EUR | | 1999-10-29T01:26:51Z - Sell - 0.05 BTC @ 35000 USDT | trades: 1555555555 OK, 45 KO |
+| | | | | | | 1999-10-29T01:26:51Z - Sell - 1.7 BTC @ 50000 USDT | |
++----------+----------------------+---------+-----------+---------------+---------------+------------------------------------------------------+------------------------------+
+)";
+ expectStr(kExpected);
+}
+
+TEST_F(QueryResultPrinterReplayTest, EmptyJson) {
+ basicQueryResultPrinter(ApiOutputType::kJson)
+ .printMarketTradingResults(timeWindow, MarketTradingGlobalResultPerExchange{}, commandType);
+ static constexpr std::string_view kExpected = R"json(
+{
+ "in": {
+ "opt": {
+ "time-window": "[1999-03-25 04:46:43 -> 2000-10-07 01:14:27)"
+ },
+ "req": "Replay"
+ },
+ "out": {}
+})json";
+ expectJson(kExpected);
+}
+
+TEST_F(QueryResultPrinterReplayTest, Json) {
+ basicQueryResultPrinter(ApiOutputType::kJson)
+ .printMarketTradingResults(timeWindow, marketTradingResultPerExchange, commandType);
+ static constexpr std::string_view kExpected = R"json(
+{
+ "in": {
+ "opt": {
+ "time-window": "[1999-03-25 04:46:43 -> 2000-10-07 01:14:27)"
+ },
+ "req": "Replay"
+ },
+ "out": {
+ "binance": {
+ "algorithm": "test-algo",
+ "market": "BTC-EUR",
+ "matched-orders": [],
+ "profit-and-loss": "0 EUR",
+ "start-amounts": {
+ "base": "1 BTC",
+ "quote": "1000 EUR"
+ },
+ "stats": {
+ "order-books": {
+ "nb-error": 0,
+ "nb-successful": 42
+ },
+ "trades": {
+ "nb-error": 10,
+ "nb-successful": 3
+ }
+ }
+ },
+ "huobi": {
+ "algorithm": "test-algo",
+ "market": "BTC-EUR",
+ "matched-orders": [
+ {
+ "id": "1",
+ "matched": "1.5",
+ "matchedTime": "1999-03-25T04:46:43Z",
+ "pair": "BTC-USDT",
+ "placedTime": "1999-03-25T04:46:43Z",
+ "price": "35000",
+ "side": "Buy"
+ },
+ {
+ "id": "5",
+ "matched": "0.036",
+ "matchedTime": "2000-10-07T01:14:27Z",
+ "pair": "BTC-USDT",
+ "placedTime": "2000-06-11T23:58:40Z",
+ "price": "47899",
+ "side": "Sell"
+ }
+ ],
+ "profit-and-loss": "500 EUR",
+ "start-amounts": {
+ "base": "1 BTC",
+ "quote": "1000 EUR"
+ },
+ "stats": {
+ "order-books": {
+ "nb-error": 2,
+ "nb-successful": 500000
+ },
+ "trades": {
+ "nb-error": 0,
+ "nb-successful": 0
+ }
+ }
+ }
+ }
+})json";
+ expectJson(kExpected);
+}
+
+TEST_F(QueryResultPrinterReplayTest, NoPrint) {
+ basicQueryResultPrinter(ApiOutputType::kNoPrint)
+ .printMarketTradingResults(timeWindow, marketTradingResultPerExchange, commandType);
+ expectNoStr();
+}
+
} // namespace cct
diff --git a/src/engine/test/replay-algorithm-name-iterator_test.cpp b/src/engine/test/replay-algorithm-name-iterator_test.cpp
new file mode 100644
index 00000000..ee930a0b
--- /dev/null
+++ b/src/engine/test/replay-algorithm-name-iterator_test.cpp
@@ -0,0 +1,106 @@
+#include "replay-algorithm-name-iterator.hpp"
+
+#include
+
+#include
+
+#include "cct_exception.hpp"
+
+namespace cct {
+class ReplayAlgorithmNameIteratorTest : public ::testing::Test {
+ protected:
+ static constexpr std::string_view kInvalidAlgorithmNames[] = {"any", "so-what,"};
+ static constexpr std::string_view kAlgorithmNames[] = {"any", "so-what", "angry",
+ "bird", "Jack", "a-more-complex algorithm Name"};
+};
+
+TEST_F(ReplayAlgorithmNameIteratorTest, AlgorithmNamesValidity) {
+ EXPECT_THROW(ReplayAlgorithmNameIterator("", kInvalidAlgorithmNames), exception);
+ EXPECT_NO_THROW(ReplayAlgorithmNameIterator("", kAlgorithmNames));
+}
+
+TEST_F(ReplayAlgorithmNameIteratorTest, IteratorWithAll) {
+ ReplayAlgorithmNameIterator it("", kAlgorithmNames);
+
+ int algorithmPos = 0;
+ while (it.hasNext()) {
+ auto next = it.next();
+
+ switch (algorithmPos) {
+ case 0:
+ [[fallthrough]];
+ case 1:
+ [[fallthrough]];
+ case 2:
+ [[fallthrough]];
+ case 3:
+ [[fallthrough]];
+ case 4:
+ [[fallthrough]];
+ case 5:
+ EXPECT_EQ(next, kAlgorithmNames[algorithmPos]);
+ break;
+ default:
+ throw exception("Unexpected number of algorithm names");
+ }
+
+ ++algorithmPos;
+ }
+
+ EXPECT_EQ(algorithmPos, 6);
+}
+
+TEST_F(ReplayAlgorithmNameIteratorTest, IteratorWithUniqueAlgorithmSpecified) {
+ ReplayAlgorithmNameIterator it("so-What", kAlgorithmNames);
+
+ int algorithmPos = 0;
+ while (it.hasNext()) {
+ auto next = it.next();
+
+ switch (algorithmPos) {
+ case 0:
+ EXPECT_EQ(next, "so-What");
+ break;
+ default:
+ throw exception("Unexpected number of algorithm names");
+ }
+
+ ++algorithmPos;
+ }
+
+ EXPECT_EQ(algorithmPos, 1);
+}
+
+TEST_F(ReplayAlgorithmNameIteratorTest, IteratorWithSpecifiedList) {
+ ReplayAlgorithmNameIterator it("Jack,whatever,so-what,some-algorithmNameThatIsNotInAll,with spaces", kAlgorithmNames);
+
+ int algorithmPos = 0;
+ while (it.hasNext()) {
+ auto next = it.next();
+
+ switch (algorithmPos) {
+ case 0:
+ EXPECT_EQ(next, "Jack");
+ break;
+ case 1:
+ EXPECT_EQ(next, "whatever");
+ break;
+ case 2:
+ EXPECT_EQ(next, "so-what");
+ break;
+ case 3:
+ EXPECT_EQ(next, "some-algorithmNameThatIsNotInAll");
+ break;
+ case 4:
+ EXPECT_EQ(next, "with spaces");
+ break;
+ default:
+ throw exception("Unexpected number of algorithm names");
+ }
+
+ ++algorithmPos;
+ }
+
+ EXPECT_EQ(algorithmPos, 5);
+}
+} // namespace cct
\ No newline at end of file
diff --git a/src/engine/test/stringoptionparser_test.cpp b/src/engine/test/stringoptionparser_test.cpp
index e8869af4..4059145b 100644
--- a/src/engine/test/stringoptionparser_test.cpp
+++ b/src/engine/test/stringoptionparser_test.cpp
@@ -2,6 +2,7 @@
#include
+#include
#include
#include "cct_invalid_argument_exception.hpp"
@@ -12,6 +13,7 @@
#include "exchangename.hpp"
#include "market.hpp"
#include "monetaryamount.hpp"
+#include "timedef.hpp"
namespace cct {
namespace {
@@ -211,4 +213,26 @@ TEST(StringOptionParserTest, ExchangesNotLast) {
EXPECT_NO_THROW(parser.checkEndParsing());
}
+TEST(StringOptionParserTest, ParseDurationMandatory) {
+ StringOptionParser parser(" 45min83s,kraken,upbit");
+
+ EXPECT_EQ(parser.parseDuration(StringOptionParser::FieldIs::kMandatory),
+ std::chrono::minutes{45} + std::chrono::seconds{83});
+ EXPECT_EQ(parser.parseExchanges(',', '\0'), ExchangeNames({ExchangeName("kraken"), ExchangeName("upbit")}));
+
+ EXPECT_NO_THROW(parser.checkEndParsing());
+}
+
+TEST(StringOptionParserTest, ParseDurationOptional) {
+ StringOptionParser parser("binance,huobi_user1,34h 4500ms");
+
+ EXPECT_EQ(parser.parseDuration(StringOptionParser::FieldIs::kOptional), kUndefinedDuration);
+ EXPECT_EQ(parser.parseExchanges(',', '\0'), ExchangeNames({ExchangeName("binance"), ExchangeName("huobi", "user1")}));
+
+ EXPECT_EQ(parser.parseDuration(StringOptionParser::FieldIs::kOptional),
+ std::chrono::hours{34} + std::chrono::milliseconds{4500});
+
+ EXPECT_NO_THROW(parser.checkEndParsing());
+}
+
} // namespace cct
\ No newline at end of file
diff --git a/src/objects/CMakeLists.txt b/src/objects/CMakeLists.txt
index 7bacf80b..385bbe35 100644
--- a/src/objects/CMakeLists.txt
+++ b/src/objects/CMakeLists.txt
@@ -99,6 +99,15 @@ add_unit_test(
CCT_DISABLE_SPDLOG
)
+add_unit_test(
+ time-window_test
+ test/time-window_test.cpp
+ LIBRARIES
+ coincenter_objects
+ DEFINITIONS
+ CCT_DISABLE_SPDLOG
+)
+
add_unit_test(
wallet_test
test/wallet_test.cpp
diff --git a/src/objects/include/automation-config.hpp b/src/objects/include/automation-config.hpp
new file mode 100644
index 00000000..669bb8dc
--- /dev/null
+++ b/src/objects/include/automation-config.hpp
@@ -0,0 +1,30 @@
+#pragma once
+
+#include
+
+#include "monetaryamount.hpp"
+#include "timedef.hpp"
+
+namespace cct {
+class AutomationConfig {
+ public:
+ AutomationConfig() noexcept = default;
+
+ AutomationConfig(Duration loadChunkDuration, MonetaryAmount startBaseAmountEquivalent,
+ MonetaryAmount startQuoteAmountEquivalent)
+ : _loadChunkDuration(loadChunkDuration),
+ _startBaseAmountEquivalent(startBaseAmountEquivalent),
+ _startQuoteAmountEquivalent(startQuoteAmountEquivalent) {}
+
+ Duration loadChunkDuration() const { return _loadChunkDuration; }
+
+ MonetaryAmount startBaseAmountEquivalent() const { return _startBaseAmountEquivalent; }
+
+ MonetaryAmount startQuoteAmountEquivalent() const { return _startQuoteAmountEquivalent; }
+
+ private:
+ Duration _loadChunkDuration = std::chrono::weeks(1);
+ MonetaryAmount _startBaseAmountEquivalent;
+ MonetaryAmount _startQuoteAmountEquivalent;
+};
+} // namespace cct
\ No newline at end of file
diff --git a/src/objects/include/coincentercommandtype.hpp b/src/objects/include/coincentercommandtype.hpp
index cc1002dc..49967dc6 100644
--- a/src/objects/include/coincentercommandtype.hpp
+++ b/src/objects/include/coincentercommandtype.hpp
@@ -31,6 +31,10 @@ enum class CoincenterCommandType : int8_t {
kWithdrawApply,
kDustSweeper,
+ kMarketData,
+ kReplay,
+ kReplayMarkets,
+
kLast
};
@@ -39,4 +43,4 @@ std::string_view CoincenterCommandTypeToString(CoincenterCommandType type);
CoincenterCommandType CoincenterCommandTypeFromString(std::string_view str);
bool IsAnyTrade(CoincenterCommandType type);
-} // namespace cct
\ No newline at end of file
+} // namespace cct
diff --git a/src/objects/include/exchange-names.hpp b/src/objects/include/exchange-names.hpp
index 339a5e18..b9850579 100644
--- a/src/objects/include/exchange-names.hpp
+++ b/src/objects/include/exchange-names.hpp
@@ -2,6 +2,8 @@
#include
+#include "cct_const.hpp"
+#include "cct_fixedcapacityvector.hpp"
#include "cct_smallvector.hpp"
#include "cct_string.hpp"
#include "exchangename.hpp"
@@ -11,6 +13,8 @@ namespace cct {
using ExchangeNameSpan = std::span;
using ExchangeNames = SmallVector;
+using PublicExchangeNameVector = FixedCapacityVector;
+
string ConstructAccumulatedExchangeNames(ExchangeNameSpan exchangeNames);
} // namespace cct
\ No newline at end of file
diff --git a/src/objects/include/exchangeconfig.hpp b/src/objects/include/exchangeconfig.hpp
index 17c9375c..008e77cf 100644
--- a/src/objects/include/exchangeconfig.hpp
+++ b/src/objects/include/exchangeconfig.hpp
@@ -19,7 +19,8 @@
namespace cct {
class ExchangeConfig {
public:
- enum struct FeeType { kMaker, kTaker };
+ enum class FeeType : int8_t { kMaker, kTaker };
+ enum class MarketDataSerialization : int8_t { kYes, kNo };
struct APIUpdateFrequencies {
Duration freq[api::kQueryTypeMax];
@@ -32,7 +33,8 @@ class ExchangeConfig {
std::string_view acceptEncoding, int dustSweeperMaxNbTrades,
log::level::level_enum requestsCallLogLevel, log::level::level_enum requestsAnswerLogLevel,
bool multiTradeAllowedByDefault, bool validateDepositAddressesInFile, bool placeSimulateRealOrder,
- bool validateApiKey, TradeConfig tradeConfig, HttpConfig httpConfig);
+ bool validateApiKey, TradeConfig tradeConfig, HttpConfig httpConfig,
+ MarketDataSerialization marketDataSerialization);
/// Get a reference to the list of statically excluded currency codes to consider for the exchange,
/// In both trading and withdrawal.
@@ -107,6 +109,8 @@ class ExchangeConfig {
PermanentCurlOptions::Builder curlOptionsBuilderBase(Api api) const;
+ bool withMarketDataSerialization() const { return _withMarketSerialization; }
+
private:
CurrencyCodeSet _excludedCurrenciesAll; // Currencies will be completely ignored by the exchange
CurrencyCodeSet _excludedCurrenciesWithdrawal; // Currencies unavailable for withdrawals
@@ -128,5 +132,6 @@ class ExchangeConfig {
bool _validateDepositAddressesInFile;
bool _placeSimulateRealOrder;
bool _validateApiKey;
+ bool _withMarketSerialization;
};
} // namespace cct
diff --git a/src/objects/include/generalconfig.hpp b/src/objects/include/generalconfig.hpp
index 0fd87678..1a30b897 100644
--- a/src/objects/include/generalconfig.hpp
+++ b/src/objects/include/generalconfig.hpp
@@ -7,6 +7,7 @@
#include "logginginfo.hpp"
#include "requestsconfig.hpp"
#include "timedef.hpp"
+#include "trading-config.hpp"
namespace cct {
@@ -18,13 +19,15 @@ class GeneralConfig {
GeneralConfig() = default;
- GeneralConfig(LoggingInfo &&loggingInfo, RequestsConfig &&requestsConfig, Duration fiatConversionQueryRate,
- ApiOutputType apiOutputType);
+ GeneralConfig(LoggingInfo &&loggingInfo, RequestsConfig &&requestsConfig, TradingConfig &&tradingConfig,
+ Duration fiatConversionQueryRate, ApiOutputType apiOutputType);
const LoggingInfo &loggingInfo() const { return _loggingInfo; }
const RequestsConfig &requestsConfig() const { return _requestsConfig; }
+ const TradingConfig &tradingConfig() const { return _tradingConfig; }
+
ApiOutputType apiOutputType() const { return _apiOutputType; }
Duration fiatConversionQueryRate() const { return _fiatConversionQueryRate; }
@@ -32,6 +35,7 @@ class GeneralConfig {
private:
LoggingInfo _loggingInfo{LoggingInfo::WithLoggersCreation::kYes};
RequestsConfig _requestsConfig;
+ TradingConfig _tradingConfig;
Duration _fiatConversionQueryRate = std::chrono::hours(8);
ApiOutputType _apiOutputType = ApiOutputType::kFormattedTable;
};
diff --git a/src/objects/include/generalconfigdefault.hpp b/src/objects/include/generalconfigdefault.hpp
index b4e6b7ef..460e0ec5 100644
--- a/src/objects/include/generalconfigdefault.hpp
+++ b/src/objects/include/generalconfigdefault.hpp
@@ -35,6 +35,17 @@ struct GeneralConfigDefault {
"concurrency": {
"nbMaxParallelRequests": 1
}
+ },
+ "trading": {
+ "automation": {
+ "deserialization": {
+ "loadChunkDuration": "1w"
+ },
+ "startingContext": {
+ "startBaseAmountEquivalent": "1000 EUR",
+ "startQuoteAmountEquivalent": "1000 EUR"
+ }
+ }
}
}
)"_json;
diff --git a/src/objects/include/marketorderbook.hpp b/src/objects/include/marketorderbook.hpp
index f72a790b..7d74f5aa 100644
--- a/src/objects/include/marketorderbook.hpp
+++ b/src/objects/include/marketorderbook.hpp
@@ -50,6 +50,8 @@ class MarketOrderBook {
int size() const { return _orders.size(); }
/// Check if data stored in this MarketOrderBook is valid.
+ /// This is especially useful for optional check of data after deserialization,
+ /// as for the standard case the market order book should be valid by design.
bool isValid() const;
bool isArtificiallyExtended() const { return _isArtificiallyExtended; }
@@ -192,6 +194,12 @@ class MarketOrderBook {
/// 0.35 20
/// 0.34 23
+ // To allow faster MarketOrderBook constructs
+ friend class MarketOrderBookConverter;
+
+ MarketOrderBook(TimePoint timeStamp, Market market, AmountPriceVector&& orders, int32_t highestBidPricePos,
+ int32_t lowestAskPricePos, VolAndPriNbDecimals volAndPriNbDecimals);
+
MonetaryAmount amountAt(int pos) const {
return MonetaryAmount(_orders[pos].amount, _market.base(), _volAndPriNbDecimals.volNbDecimals);
}
diff --git a/src/objects/include/time-window.hpp b/src/objects/include/time-window.hpp
new file mode 100644
index 00000000..4d641d90
--- /dev/null
+++ b/src/objects/include/time-window.hpp
@@ -0,0 +1,70 @@
+#pragma once
+
+#include
+#include
+
+#include "cct_format.hpp"
+#include "cct_invalid_argument_exception.hpp"
+#include "timedef.hpp"
+
+namespace cct {
+
+/// Simple utility class representing a time window with a beginning time, and an end time.
+/// The beginning time includes the corresponding time point, but the end time excludes it.
+class TimeWindow {
+ public:
+ /// Create a zero duration time window starting from the zero-initialized time point.
+ TimeWindow() noexcept = default;
+
+ /// Create a time window spanning from 'from' (included) to 'to' (excluded) time points.
+ TimeWindow(TimePoint from, TimePoint to) : _from(from), _to(to) {
+ if (_to < _from) {
+ throw invalid_argument("Invalid time window - 'from' should not be larger than 'to'");
+ }
+ }
+
+ /// Create a time window starting at 'from' with 'dur' duration.
+ TimeWindow(TimePoint from, Duration dur) : TimeWindow(from, from + dur) {}
+
+ TimePoint from() const { return _from; }
+
+ TimePoint to() const { return _to; }
+
+ Duration duration() const { return _to - _from; }
+
+ bool contains(TimePoint tp) const { return _from <= tp && tp < _to; }
+
+ bool contains(int64_t unixTimestampInMs) const { return contains(TimePoint(milliseconds{unixTimestampInMs})); }
+
+ bool contains(TimeWindow rhs) const { return _from <= rhs._from && rhs._to <= _to; }
+
+ bool overlaps(TimeWindow rhs) const { return _from < rhs._to && rhs._from < _to; }
+
+ string str() const;
+
+ bool operator==(const TimeWindow&) const noexcept = default;
+
+ private:
+ TimePoint _from;
+ TimePoint _to;
+};
+} // namespace cct
+
+#ifndef CCT_DISABLE_SPDLOG
+template <>
+struct fmt::formatter {
+ constexpr auto parse(format_parse_context& ctx) -> decltype(ctx.begin()) {
+ const auto it = ctx.begin();
+ const auto end = ctx.end();
+ if (it != end && *it != '}') {
+ throw format_error("invalid format");
+ }
+ return it;
+ }
+
+ template
+ auto format(const cct::TimeWindow& timeWindow, FormatContext& ctx) const -> decltype(ctx.out()) {
+ return fmt::format_to(ctx.out(), "{}", timeWindow.str());
+ }
+};
+#endif
diff --git a/src/objects/include/trading-config.hpp b/src/objects/include/trading-config.hpp
new file mode 100644
index 00000000..b265e9dc
--- /dev/null
+++ b/src/objects/include/trading-config.hpp
@@ -0,0 +1,19 @@
+#pragma once
+
+#include
+
+#include "automation-config.hpp"
+
+namespace cct {
+class TradingConfig {
+ public:
+ TradingConfig() noexcept = default;
+
+ TradingConfig(AutomationConfig automationConfig) : _automationConfig(std::move(automationConfig)) {}
+
+ const AutomationConfig &automationConfig() const { return _automationConfig; }
+
+ private:
+ AutomationConfig _automationConfig;
+};
+} // namespace cct
\ No newline at end of file
diff --git a/src/objects/src/coincentercommandtype.cpp b/src/objects/src/coincentercommandtype.cpp
index 8ec46ee1..bbb481ef 100644
--- a/src/objects/src/coincentercommandtype.cpp
+++ b/src/objects/src/coincentercommandtype.cpp
@@ -16,8 +16,7 @@ constexpr std::string_view kCommandTypeNames[] = {
"Balance", "DepositInfo", "OrdersClosed", "OrdersOpened", "OrdersCancel",
"RecentDeposits", "RecentWithdraws", "Trade", "Buy", "Sell",
- "Withdraw", "DustSweeper",
-};
+ "Withdraw", "DustSweeper", "MarketData", "Replay", "ReplayMarkets"};
static_assert(std::size(kCommandTypeNames) == static_cast(CoincenterCommandType::kLast));
} // namespace
@@ -51,4 +50,4 @@ bool IsAnyTrade(CoincenterCommandType type) {
return false;
}
}
-} // namespace cct
\ No newline at end of file
+} // namespace cct
diff --git a/src/objects/src/exchangeconfig.cpp b/src/objects/src/exchangeconfig.cpp
index 61c1b4ab..ce9e8406 100644
--- a/src/objects/src/exchangeconfig.cpp
+++ b/src/objects/src/exchangeconfig.cpp
@@ -67,7 +67,8 @@ ExchangeConfig::ExchangeConfig(
const APIUpdateFrequencies &apiUpdateFrequencies, Duration publicAPIRate, Duration privateAPIRate,
std::string_view acceptEncoding, int dustSweeperMaxNbTrades, log::level::level_enum requestsCallLogLevel,
log::level::level_enum requestsAnswerLogLevel, bool multiTradeAllowedByDefault, bool validateDepositAddressesInFile,
- bool placeSimulateRealOrder, bool validateApiKey, TradeConfig tradeConfig, HttpConfig httpConfig)
+ bool placeSimulateRealOrder, bool validateApiKey, TradeConfig tradeConfig, HttpConfig httpConfig,
+ MarketDataSerialization marketDataSerialization)
: _excludedCurrenciesAll(std::move(excludedAllCurrencies)),
_excludedCurrenciesWithdrawal(std::move(excludedCurrenciesWithdraw)),
_preferredPaymentCurrencies(std::move(preferredPaymentCurrencies)),
@@ -86,7 +87,8 @@ ExchangeConfig::ExchangeConfig(
_multiTradeAllowedByDefault(multiTradeAllowedByDefault),
_validateDepositAddressesInFile(validateDepositAddressesInFile),
_placeSimulateRealOrder(placeSimulateRealOrder),
- _validateApiKey(validateApiKey) {
+ _validateApiKey(validateApiKey),
+ _withMarketSerialization(marketDataSerialization == MarketDataSerialization::kYes) {
if (dustSweeperMaxNbTrades > std::numeric_limits::max() || dustSweeperMaxNbTrades < 0) {
throw exception("Invalid number of dust sweeper max trades '{}', should be in [0, {}]", dustSweeperMaxNbTrades,
std::numeric_limits::max());
@@ -113,6 +115,7 @@ ExchangeConfig::ExchangeConfig(
_validateDepositAddressesInFile ? kDepositAddressesFileName : "");
log::trace(" - Order placing in simulation : {}", _placeSimulateRealOrder ? "real, unmatchable" : "none");
log::trace(" - Validate API Key : {}", _validateApiKey ? "yes" : "no");
+ log::trace(" - Market data serialization : {}", _withMarketSerialization ? "yes" : "no");
}
if (_preferredPaymentCurrencies.empty()) {
log::warn("{} list of preferred currencies is empty, buy and sell commands cannot perform trades", exchangeNameStr);
diff --git a/src/objects/src/exchangeconfigdefault.hpp b/src/objects/src/exchangeconfigdefault.hpp
index 3199cbd1..93e48258 100644
--- a/src/objects/src/exchangeconfigdefault.hpp
+++ b/src/objects/src/exchangeconfigdefault.hpp
@@ -59,6 +59,7 @@ struct ExchangeConfigDefault {
"requestsCall": "info",
"requestsAnswer": "trace"
},
+ "marketDataSerialization": true,
"multiTradeAllowedByDefault": false,
"placeSimulateRealOrder": false,
"trade": {
@@ -186,6 +187,7 @@ struct ExchangeConfigDefault {
"requestsCall": "info",
"requestsAnswer": "trace"
},
+ "marketDataSerialization": false,
"multiTradeAllowedByDefault": true,
"privateAPIRate": "1055ms",
"publicAPIRate": "1236ms",
diff --git a/src/objects/src/exchangeconfigmap.cpp b/src/objects/src/exchangeconfigmap.cpp
index d9b5a472..16f9e6f6 100644
--- a/src/objects/src/exchangeconfigmap.cpp
+++ b/src/objects/src/exchangeconfigmap.cpp
@@ -58,6 +58,10 @@ ExchangeConfigMap ComputeExchangeConfigMap(std::string_view fileName, const json
withdrawTopLevelOption.getBool(exchangeName, "validateDepositAddressesInFile");
const bool placeSimulatedRealOrder = queryTopLevelOption.getBool(exchangeName, "placeSimulateRealOrder");
const bool validateApiKey = queryTopLevelOption.getBool(exchangeName, "validateApiKey");
+ const ExchangeConfig::MarketDataSerialization marketDataSerialization =
+ queryTopLevelOption.getBool(exchangeName, "marketDataSerialization")
+ ? ExchangeConfig::MarketDataSerialization::kYes
+ : ExchangeConfig::MarketDataSerialization::kNo;
MonetaryAmountByCurrencySet dustAmountsThresholds(
queryTopLevelOption.getMonetaryAmountsArray(exchangeName, "dustAmountsThreshold"));
@@ -90,7 +94,7 @@ ExchangeConfigMap ComputeExchangeConfigMap(std::string_view fileName, const json
std::move(dustAmountsThresholds), std::move(apiUpdateFrequencies), publicAPIRate, privateAPIRate,
acceptEncoding, dustSweeperMaxNbTrades, requestsCallLogLevel, requestsAnswerLogLevel,
multiTradeAllowedByDefault, validateDepositAddressesInFile, placeSimulatedRealOrder,
- validateApiKey, std::move(tradeConfig), std::move(httpConfig)));
+ validateApiKey, std::move(tradeConfig), std::move(httpConfig), marketDataSerialization));
} // namespace cct
// Print json unused values
@@ -120,4 +124,4 @@ ExchangeConfigMap ComputeExchangeConfigMap(std::string_view fileName, const json
return map;
}
-} // namespace cct
\ No newline at end of file
+} // namespace cct
diff --git a/src/objects/src/generalconfig.cpp b/src/objects/src/generalconfig.cpp
index c08c3270..96494ec1 100644
--- a/src/objects/src/generalconfig.cpp
+++ b/src/objects/src/generalconfig.cpp
@@ -11,13 +11,15 @@
#include "logginginfo.hpp"
#include "requestsconfig.hpp"
#include "timedef.hpp"
+#include "trading-config.hpp"
namespace cct {
-GeneralConfig::GeneralConfig(LoggingInfo &&loggingInfo, RequestsConfig &&requestsConfig,
+GeneralConfig::GeneralConfig(LoggingInfo &&loggingInfo, RequestsConfig &&requestsConfig, TradingConfig &&tradingConfig,
Duration fiatConversionQueryRate, ApiOutputType apiOutputType)
: _loggingInfo(std::move(loggingInfo)),
_requestsConfig(std::move(requestsConfig)),
+ _tradingConfig(std::move(tradingConfig)),
_fiatConversionQueryRate(fiatConversionQueryRate),
_apiOutputType(apiOutputType) {}
diff --git a/src/objects/src/marketorderbook.cpp b/src/objects/src/marketorderbook.cpp
index 89791843..b4fd14fa 100644
--- a/src/objects/src/marketorderbook.cpp
+++ b/src/objects/src/marketorderbook.cpp
@@ -236,6 +236,16 @@ MarketOrderBook::MarketOrderBook(TimePoint timeStamp, MonetaryAmount askPrice, M
}
}
+MarketOrderBook::MarketOrderBook(TimePoint timeStamp, Market market, AmountPriceVector&& orders,
+ int32_t highestBidPricePos, int32_t lowestAskPricePos,
+ VolAndPriNbDecimals volAndPriNbDecimals)
+ : _time(timeStamp),
+ _market(market),
+ _orders(std::move(orders)),
+ _highestBidPricePos(highestBidPricePos),
+ _lowestAskPricePos(lowestAskPricePos),
+ _volAndPriNbDecimals(volAndPriNbDecimals) {}
+
bool MarketOrderBook::isValid() const {
if (_orders.size() < 2U) {
log::error("Market order book is invalid as size is {}", _orders.size());
diff --git a/src/objects/src/publictrade.cpp b/src/objects/src/publictrade.cpp
index dd023f84..c1cc9ba0 100644
--- a/src/objects/src/publictrade.cpp
+++ b/src/objects/src/publictrade.cpp
@@ -34,4 +34,4 @@ bool PublicTrade::isValid() const {
return true;
}
-} // namespace cct
\ No newline at end of file
+} // namespace cct
diff --git a/src/objects/src/time-window.cpp b/src/objects/src/time-window.cpp
new file mode 100644
index 00000000..088281f6
--- /dev/null
+++ b/src/objects/src/time-window.cpp
@@ -0,0 +1,15 @@
+#include "time-window.hpp"
+
+#include "timestring.hpp"
+
+namespace cct {
+string TimeWindow::str() const {
+ string ret;
+ ret.push_back('[');
+ ret.append(ToString(from(), kTimeYearToSecondSpaceSeparatedFormat));
+ ret.append(" -> ");
+ ret.append(ToString(to(), kTimeYearToSecondSpaceSeparatedFormat));
+ ret.push_back(')');
+ return ret;
+}
+} // namespace cct
\ No newline at end of file
diff --git a/src/objects/test/time-window_test.cpp b/src/objects/test/time-window_test.cpp
new file mode 100644
index 00000000..42f6b04a
--- /dev/null
+++ b/src/objects/test/time-window_test.cpp
@@ -0,0 +1,140 @@
+#include "time-window.hpp"
+
+#include
+
+#include
+
+#include "cct_invalid_argument_exception.hpp"
+#include "timedef.hpp"
+
+namespace cct {
+class TimeWindowTest : public ::testing::Test {
+ protected:
+ TimePoint tp1{milliseconds{std::numeric_limits::max() / 10000000}};
+ TimePoint tp2{milliseconds{std::numeric_limits::max() / 9900000}};
+ TimePoint tp3{milliseconds{std::numeric_limits::max() / 9800000}};
+ TimePoint tp4{milliseconds{std::numeric_limits::max() / 9500000}};
+ TimePoint tp5{milliseconds{std::numeric_limits::max() / 9000000}};
+
+ Duration dur1{seconds{100}};
+ Duration dur2{seconds{1000}};
+ Duration dur3{seconds{10000}};
+};
+
+TEST_F(TimeWindowTest, DefaultConstructor) {
+ TimeWindow tw;
+
+ EXPECT_EQ(tw.from(), TimePoint{});
+ EXPECT_EQ(tw.to(), TimePoint{});
+ EXPECT_EQ(tw.duration(), milliseconds{});
+ EXPECT_FALSE(tw.contains(TimePoint{}));
+ EXPECT_FALSE(tw.contains(0));
+ EXPECT_TRUE(tw.contains(tw));
+}
+
+TEST_F(TimeWindowTest, InvalidTimeWindowFromTime) { EXPECT_THROW(TimeWindow(tp2, tp1), invalid_argument); }
+TEST_F(TimeWindowTest, InvalidTimeWindowFromDuration) { EXPECT_THROW(TimeWindow(tp1, tp1 - tp2), invalid_argument); }
+
+TEST_F(TimeWindowTest, DurationConstructor) {
+ TimeWindow tw(tp1, tp2 - tp1);
+
+ EXPECT_EQ(tw, TimeWindow(tp1, tp2));
+}
+
+TEST_F(TimeWindowTest, Duration) {
+ TimeWindow tw(tp1, tp2);
+
+ EXPECT_EQ(tw.duration(), tp2 - tp1);
+}
+
+TEST_F(TimeWindowTest, ContainsTimePoint) {
+ TimeWindow tw1(tp1, tp2);
+
+ EXPECT_TRUE(tw1.contains(tp1));
+ EXPECT_TRUE(tw1.contains(tp1 + dur1));
+ EXPECT_FALSE(tw1.contains(tp2));
+ EXPECT_FALSE(tw1.contains(tp3));
+}
+
+TEST_F(TimeWindowTest, ContainsTimeWindow) {
+ // [ ]
+ // [ ]
+ TimeWindow tw1(tp1, tp4);
+ TimeWindow tw2(tp2, tp3);
+
+ EXPECT_TRUE(tw1.contains(tw1));
+ EXPECT_TRUE(tw1.overlaps(tw1));
+
+ EXPECT_TRUE(tw1.overlaps(tw2));
+ EXPECT_TRUE(tw1.contains(tw2));
+
+ EXPECT_TRUE(tw2.overlaps(tw1));
+ EXPECT_FALSE(tw2.contains(tw1));
+}
+
+TEST_F(TimeWindowTest, OverlapNominal) {
+ // [ ]
+ // [ ]
+ TimeWindow tw1(tp2, tp4);
+ TimeWindow tw2(tp1, tp3);
+
+ EXPECT_TRUE(tw1.overlaps(tw2));
+ EXPECT_FALSE(tw1.contains(tw2));
+
+ EXPECT_TRUE(tw2.overlaps(tw1));
+ EXPECT_FALSE(tw2.contains(tw1));
+}
+
+TEST_F(TimeWindowTest, OverlapEqualTo) {
+ // [ ]
+ // [ ]
+ TimeWindow tw1(tp1, tp3);
+ TimeWindow tw2(tp2, tp3);
+
+ EXPECT_TRUE(tw1.overlaps(tw2));
+ EXPECT_TRUE(tw1.contains(tw2));
+
+ EXPECT_TRUE(tw2.overlaps(tw1));
+ EXPECT_FALSE(tw2.contains(tw1));
+}
+
+TEST_F(TimeWindowTest, OverlapEqualFrom) {
+ // [ ]
+ // [ ]
+ TimeWindow tw1(tp1, tp3);
+ TimeWindow tw2(tp1, tp2);
+
+ EXPECT_TRUE(tw1.overlaps(tw2));
+ EXPECT_TRUE(tw1.contains(tw2));
+
+ EXPECT_TRUE(tw2.overlaps(tw1));
+ EXPECT_FALSE(tw2.contains(tw1));
+}
+
+TEST_F(TimeWindowTest, NoOverlapNominal) {
+ // [ ]
+ // [ ]
+ TimeWindow tw1(tp1, tp2);
+ TimeWindow tw2(tp3, tp4);
+
+ EXPECT_FALSE(tw1.overlaps(tw2));
+ EXPECT_FALSE(tw1.contains(tw2));
+
+ EXPECT_FALSE(tw2.overlaps(tw1));
+ EXPECT_FALSE(tw2.contains(tw1));
+}
+
+TEST_F(TimeWindowTest, NoOverlapEqual) {
+ // [ ]
+ // [ ]
+ TimeWindow tw1(tp1, tp3);
+ TimeWindow tw2(tp3, tp4);
+
+ EXPECT_FALSE(tw1.overlaps(tw2));
+ EXPECT_FALSE(tw1.contains(tw2));
+
+ EXPECT_FALSE(tw2.overlaps(tw1));
+ EXPECT_FALSE(tw2.contains(tw1));
+}
+
+} // namespace cct
diff --git a/src/serialization/CMakeLists.txt b/src/serialization/CMakeLists.txt
new file mode 100644
index 00000000..1724597e
--- /dev/null
+++ b/src/serialization/CMakeLists.txt
@@ -0,0 +1,74 @@
+if(CCT_ENABLE_PROTO)
+ aux_source_directory(src SERIALIZATION_SRC)
+
+ list(APPEND SERIALIZATION_SRC "${CMAKE_CURRENT_LIST_DIR}/proto/market-order-book.proto")
+ list(APPEND SERIALIZATION_SRC "${CMAKE_CURRENT_LIST_DIR}/proto/public-trade.proto")
+else()
+ set(SERIALIZATION_SRC "src/dummy-market-data-serializer.cpp" "src/dummy-market-data-deserializer.cpp")
+endif()
+
+add_library(coincenter_serialization STATIC ${SERIALIZATION_SRC})
+
+target_include_directories(coincenter_serialization PUBLIC include)
+target_link_libraries(coincenter_serialization PUBLIC coincenter_objects)
+
+if(CCT_ENABLE_PROTO)
+ set(PROTO_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/generated")
+
+ target_include_directories(coincenter_serialization PUBLIC "$")
+
+ target_link_libraries(coincenter_serialization PUBLIC protobuf::libprotobuf)
+
+ if(MSVC)
+ # Not sure why it's needed for Windows.
+ target_link_libraries(libprotobuf PUBLIC ZLIB::ZLIB)
+ endif()
+ target_link_libraries(coincenter_serialization PUBLIC ZLIB::ZLIB)
+
+ protobuf_generate(
+ TARGET coincenter_serialization
+ IMPORT_DIRS "${CMAKE_CURRENT_LIST_DIR}/proto"
+ PROTOC_OUT_DIR "${PROTO_BINARY_DIR}"
+ )
+
+ add_unit_test(
+ continuous-iterator_test
+ test/continuous-iterator_test.cpp
+ )
+
+ add_unit_test(
+ proto-market-order-book-converter_test
+ test/proto-market-order-book-converter_test.cpp
+ LIBRARIES
+ coincenter_serialization
+ )
+
+ add_unit_test(
+ proto-multiple-messages-handler_test
+ test/proto-multiple-messages-handler_test.cpp
+ LIBRARIES
+ coincenter_serialization
+ )
+
+ add_unit_test(
+ proto-public-trade-converter_test
+ test/proto-public-trade-converter_test.cpp
+ LIBRARIES
+ coincenter_serialization
+ )
+
+ add_unit_test(
+ proto-serialization-and-deserialization_test
+ test/proto-serialization-and-deserialization_test.cpp
+ LIBRARIES
+ coincenter_serialization
+ )
+
+ add_unit_test(
+ serialization-tools_test
+ test/serialization-tools_test.cpp
+ LIBRARIES
+ coincenter_serialization
+ )
+
+endif()
diff --git a/src/serialization/include/abstract-market-data-deserializer.hpp b/src/serialization/include/abstract-market-data-deserializer.hpp
new file mode 100644
index 00000000..fd66d9d1
--- /dev/null
+++ b/src/serialization/include/abstract-market-data-deserializer.hpp
@@ -0,0 +1,24 @@
+#pragma once
+
+#include "market-order-book-vector.hpp"
+#include "market-timestamp-set.hpp"
+#include "market.hpp"
+#include "public-trade-vector.hpp"
+#include "time-window.hpp"
+
+namespace cct {
+
+class AbstractMarketDataDeserializer {
+ public:
+ virtual ~AbstractMarketDataDeserializer() = default;
+
+ virtual MarketTimestampSet pullMarketOrderBooksMarkets(TimeWindow timeWindow) = 0;
+
+ virtual MarketTimestampSet pullTradeMarkets(TimeWindow timeWindow) = 0;
+
+ virtual MarketOrderBookVector pullMarketOrderBooks(Market market, TimeWindow timeWindow) = 0;
+
+ virtual PublicTradeVector pullTrades(Market market, TimeWindow timeWindow) = 0;
+};
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/include/abstract-market-data-serializer.hpp b/src/serialization/include/abstract-market-data-serializer.hpp
new file mode 100644
index 00000000..65cc1b79
--- /dev/null
+++ b/src/serialization/include/abstract-market-data-serializer.hpp
@@ -0,0 +1,24 @@
+#pragma once
+
+#include
+
+#include "market.hpp"
+#include "publictrade.hpp"
+
+namespace cct {
+
+class MarketOrderBook;
+
+class AbstractMarketDataSerializer {
+ public:
+ virtual ~AbstractMarketDataSerializer() = default;
+
+ /// Push market order book in the MarketDataSerializer.
+ virtual void push(const MarketOrderBook &marketOrderBook) = 0;
+
+ /// Push public trades in the MarketDataSerializer.
+ /// They should come from the same market.
+ virtual void push(Market market, std::span publicTrades) = 0;
+};
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/include/continuous-iterator.hpp b/src/serialization/include/continuous-iterator.hpp
new file mode 100644
index 00000000..91b71cd1
--- /dev/null
+++ b/src/serialization/include/continuous-iterator.hpp
@@ -0,0 +1,23 @@
+#pragma once
+
+namespace cct {
+
+/// Simple utility class that may iterate in both directions.
+class ContinuousIterator {
+ public:
+ ContinuousIterator(int from, int to) : _to(to), _curr(from), _incr(to < from ? -1 : 1) {}
+
+ bool hasNext() const { return _curr != _to + _incr; }
+
+ auto next() {
+ _curr += _incr;
+ return _curr - _incr;
+ }
+
+ private:
+ int _to;
+ int _curr;
+ int _incr;
+};
+
+} // namespace cct
diff --git a/src/serialization/include/dummy-market-data-deserializer.hpp b/src/serialization/include/dummy-market-data-deserializer.hpp
new file mode 100644
index 00000000..e3b0227c
--- /dev/null
+++ b/src/serialization/include/dummy-market-data-deserializer.hpp
@@ -0,0 +1,29 @@
+#pragma once
+
+#include
+
+#include "abstract-market-data-deserializer.hpp"
+#include "market-order-book-vector.hpp"
+#include "market-timestamp-set.hpp"
+#include "market.hpp"
+#include "public-trade-vector.hpp"
+#include "time-window.hpp"
+
+namespace cct {
+
+class DummyMarketDataDeserializer : public AbstractMarketDataDeserializer {
+ public:
+ DummyMarketDataDeserializer([[maybe_unused]] std::string_view dataDir,
+ [[maybe_unused]] std::string_view exchangeName);
+
+ MarketTimestampSet pullMarketOrderBooksMarkets([[maybe_unused]] TimeWindow timeWindow) override;
+
+ MarketTimestampSet pullTradeMarkets([[maybe_unused]] TimeWindow timeWindow) override;
+
+ MarketOrderBookVector pullMarketOrderBooks([[maybe_unused]] Market market,
+ [[maybe_unused]] TimeWindow timeWindow) override;
+
+ PublicTradeVector pullTrades([[maybe_unused]] Market market, [[maybe_unused]] TimeWindow timeWindow) override;
+};
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/include/dummy-market-data-serializer.hpp b/src/serialization/include/dummy-market-data-serializer.hpp
new file mode 100644
index 00000000..7665cae3
--- /dev/null
+++ b/src/serialization/include/dummy-market-data-serializer.hpp
@@ -0,0 +1,27 @@
+#pragma once
+
+#include
+#include
+
+#include "abstract-market-data-serializer.hpp"
+#include "market-timestamp-set.hpp"
+#include "market.hpp"
+#include "publictrade.hpp"
+
+namespace cct {
+
+class MarketOrderBook;
+
+/// Implementation of a market data serializer that does nothing.
+/// Useful if coincenter is not compiled with protobuf support.
+class DummyMarketDataSerializer : public AbstractMarketDataSerializer {
+ public:
+ DummyMarketDataSerializer(std::string_view dataDir, const MarketTimestampSets &lastWrittenObjectsMarketTimestamp,
+ std::string_view exchangeName);
+
+ void push(const MarketOrderBook &marketOrderBook) override;
+
+ void push(Market market, std::span publicTrades) override;
+};
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/include/market-timestamp-set.hpp b/src/serialization/include/market-timestamp-set.hpp
new file mode 100644
index 00000000..afda7143
--- /dev/null
+++ b/src/serialization/include/market-timestamp-set.hpp
@@ -0,0 +1,15 @@
+#pragma once
+
+#include "cct_flatset.hpp"
+#include "market-timestamp.hpp"
+
+namespace cct {
+
+using MarketTimestampSet = FlatSet;
+
+struct MarketTimestampSets {
+ MarketTimestampSet orderBooksMarkets;
+ MarketTimestampSet tradesMarkets;
+};
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/include/market-timestamp.hpp b/src/serialization/include/market-timestamp.hpp
new file mode 100644
index 00000000..67317c69
--- /dev/null
+++ b/src/serialization/include/market-timestamp.hpp
@@ -0,0 +1,17 @@
+#pragma once
+
+#include
+
+#include "market.hpp"
+#include "timedef.hpp"
+
+namespace cct {
+
+struct MarketTimestamp {
+ Market market;
+ TimePoint timePoint;
+
+ std::strong_ordering operator<=>(const MarketTimestamp &) const noexcept = default;
+};
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/include/proto-constants.hpp b/src/serialization/include/proto-constants.hpp
new file mode 100644
index 00000000..28a48461
--- /dev/null
+++ b/src/serialization/include/proto-constants.hpp
@@ -0,0 +1,15 @@
+#pragma once
+
+#include
+#include
+
+namespace cct {
+
+enum class ProtobufObject : int8_t { kMarketOrderBook, kTrade };
+
+static constexpr std::string_view kBinProtobufExtension = ".binpb";
+
+static constexpr std::string_view kSubPathMarketOrderBooks = "order-books";
+static constexpr std::string_view kSubPathTrades = "trades";
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/include/proto-deserializer.hpp b/src/serialization/include/proto-deserializer.hpp
new file mode 100644
index 00000000..0440145e
--- /dev/null
+++ b/src/serialization/include/proto-deserializer.hpp
@@ -0,0 +1,183 @@
+#pragma once
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "cct_log.hpp"
+#include "cct_vector.hpp"
+#include "continuous-iterator.hpp"
+#include "market-timestamp-set.hpp"
+#include "market-timestamp.hpp"
+#include "market.hpp"
+#include "proto-multiple-messages-handler.hpp"
+#include "serialization-tools.hpp"
+#include "stringhelpers.hpp"
+#include "time-window.hpp"
+#include "timedef.hpp"
+
+namespace cct {
+
+template
+class ProtobufObjectsDeserializer {
+ public:
+ using CoincenterObjectType = std::invoke_result_t;
+
+ explicit ProtobufObjectsDeserializer(std::filesystem::path exchangeSerializedDataPath) noexcept
+ : _exchangeSerializedDataPath(std::move(exchangeSerializedDataPath)) {}
+
+ /// Load all markets found on disk which has some data in the given time window
+ MarketTimestampSet listMarkets(TimeWindow timeWindow) {
+ vector marketTimestamps;
+
+ std::error_code ec;
+ if (std::filesystem::is_directory(_exchangeSerializedDataPath, ec)) {
+ for (const auto& marketDirectory : std::filesystem::directory_iterator(_exchangeSerializedDataPath)) {
+ auto ts = loadMarket(marketDirectory, timeWindow, ActionType::kCheckPresence).second;
+ if (ts != TimePoint{}) {
+ const auto& marketPath = marketDirectory.path();
+
+ marketTimestamps.emplace_back(Market{marketPath.filename().string()}, ts);
+ }
+ }
+ } else if (std::filesystem::exists(_exchangeSerializedDataPath)) {
+ log::error("{} is not a valid directory: {}", _exchangeSerializedDataPath.string(), ec.message());
+ }
+
+ return MarketTimestampSet(std::move(marketTimestamps));
+ }
+
+ /// Load all data found on disk for given market for the time window
+ vector loadMarket(Market market, TimeWindow timeWindow) {
+ const std::filesystem::path marketPath = _exchangeSerializedDataPath / std::string_view{market.str()};
+
+ return loadMarket(std::filesystem::directory_entry(marketPath), timeWindow, ActionType::kLoad).first;
+ }
+
+ private:
+ static bool ValidateTimestamp(const ProtobufObjType& msg, TimeWindow timeWindow) {
+ if (!msg.has_unixtimestampinms()) {
+ log::error("Invalid data loaded for protobuf object, no unix timestamp set");
+ return false;
+ }
+ return timeWindow.contains(msg.unixtimestampinms());
+ }
+
+ enum class ActionType : int8_t { kLoad, kCheckPresence };
+
+ static ContinuousIterator CreateIt(int from, int to, ActionType actionType) {
+ if (actionType == ActionType::kCheckPresence) {
+ // In check presence mode, we are only interested in the latest timestamp, so we explore towards the past
+ std::swap(from, to);
+ }
+ return {from, to};
+ }
+
+ /// Load all data found on disk for given market for the time window
+ auto loadMarket(const std::filesystem::directory_entry& marketDirectory, TimeWindow timeWindow,
+ ActionType actionType) {
+ std::pair, TimePoint> ret;
+ if (!marketDirectory.is_directory()) {
+ return ret;
+ }
+ const auto fromDays = std::chrono::floor(timeWindow.from());
+ const std::chrono::year_month_day fromYmd{fromDays};
+ const std::chrono::hh_mm_ss fromTime{std::chrono::floor(timeWindow.from() - fromDays)};
+
+ const auto toDays = std::chrono::floor(timeWindow.to());
+ const std::chrono::year_month_day toYmd{toDays};
+ const std::chrono::hh_mm_ss toTime{std::chrono::floor(timeWindow.to() - toDays)};
+
+ const auto& marketPath = marketDirectory.path();
+ const auto marketFilename = marketPath.filename();
+ const Market market(marketFilename.string());
+
+ ProtoToCoincenterObjectsFunc converter(market);
+
+ const int fromYear = static_cast(fromYmd.year());
+ const int toYear = static_cast(toYmd.year());
+
+ for (ContinuousIterator yearIt = CreateIt(fromYear, toYear, actionType); yearIt.hasNext();) {
+ const auto year = yearIt.next();
+ const auto yearPath = marketPath / std::string_view(ToCharVector(year));
+ if (!std::filesystem::is_directory(yearPath)) {
+ continue;
+ }
+ const bool isYearFromExtremity = year == fromYear;
+ const bool isYearToExtremity = year == toYear;
+ const auto fromMonth = isYearFromExtremity ? static_cast(static_cast(fromYmd.month())) : 1;
+ const auto toMonth = isYearToExtremity ? static_cast(static_cast(toYmd.month())) : 12;
+
+ for (ContinuousIterator monthIt = CreateIt(fromMonth, toMonth, actionType); monthIt.hasNext();) {
+ const auto month = monthIt.next();
+ const auto monthPath = yearPath / MonthStr(month);
+ if (!std::filesystem::is_directory(monthPath)) {
+ continue;
+ }
+ const bool isMonthFromExtremity = isYearFromExtremity && month == fromMonth;
+ const bool isMonthToExtremity = isYearToExtremity && month == toMonth;
+ const auto fromDay = isMonthFromExtremity ? static_cast(static_cast(fromYmd.day())) : 1;
+ const auto toDay = isMonthToExtremity ? static_cast(static_cast(toYmd.day())) : 31;
+
+ for (ContinuousIterator dayOfMonthIt = CreateIt(fromDay, toDay, actionType); dayOfMonthIt.hasNext();) {
+ const auto dayOfMonth = dayOfMonthIt.next();
+ const auto dayPath = monthPath / DayOfMonthStr(dayOfMonth);
+ if (!std::filesystem::is_directory(dayPath)) {
+ continue;
+ }
+
+ const bool isDayFromExtremity = isMonthFromExtremity && dayOfMonth == fromDay;
+ const bool isDayToExtremity = isMonthToExtremity && dayOfMonth == toDay;
+ const auto fromHour = isDayFromExtremity ? static_cast(fromTime.hours().count()) : 0;
+ const auto toHour = isDayToExtremity ? static_cast(toTime.hours().count()) : 23;
+
+ for (ContinuousIterator hourOfDayIt = CreateIt(fromHour, toHour, actionType); hourOfDayIt.hasNext();) {
+ const auto hourOfDay = hourOfDayIt.next();
+ const auto hourPath = dayPath / ComputeProtoFileName(hourOfDay);
+ if (!std::filesystem::exists(hourPath)) {
+ continue;
+ }
+
+ decltype(std::declval().unixtimestampinms()) lastTs = 0;
+
+ std::ifstream ifs(hourPath, std::ios::in | std::ios::binary);
+ for (ProtobufMessageCompressedReaderIterator protobufMessageReaderIt(ifs);
+ protobufMessageReaderIt.hasNext();) {
+ auto msg = protobufMessageReaderIt.next();
+ if (!ValidateTimestamp(msg, timeWindow)) {
+ continue;
+ }
+
+ // In Check presence mode, we read all the file to retrieve the latest timestamp.
+ // There's no other way to do it.
+ lastTs = msg.unixtimestampinms();
+
+ if (actionType == ActionType::kLoad) {
+ ret.first.push_back(converter(std::move(msg)));
+ }
+ }
+
+ if (lastTs != 0) {
+ if (ret.second == TimePoint{}) {
+ ret.second = TimePoint{milliseconds{static_cast(lastTs)}};
+ }
+ if (actionType == ActionType::kCheckPresence) {
+ return ret;
+ }
+ }
+ }
+ }
+ }
+ }
+ return ret;
+ }
+
+ std::filesystem::path _exchangeSerializedDataPath;
+};
+
+} // namespace cct
diff --git a/src/serialization/include/proto-market-data-deserializer.hpp b/src/serialization/include/proto-market-data-deserializer.hpp
new file mode 100644
index 00000000..2e653198
--- /dev/null
+++ b/src/serialization/include/proto-market-data-deserializer.hpp
@@ -0,0 +1,36 @@
+#pragma once
+
+#include
+
+#include "abstract-market-data-deserializer.hpp"
+#include "market-order-book-vector.hpp"
+#include "market-order-book.pb.h"
+#include "market-timestamp-set.hpp"
+#include "market.hpp"
+#include "proto-deserializer.hpp"
+#include "proto-market-order-book-converter.hpp"
+#include "proto-public-trade-converter.hpp"
+#include "public-trade-vector.hpp"
+#include "public-trade.pb.h"
+#include "time-window.hpp"
+
+namespace cct {
+
+class ProtoMarketDataDeserializer : public AbstractMarketDataDeserializer {
+ public:
+ ProtoMarketDataDeserializer(std::string_view dataDir, std::string_view exchangeName);
+
+ MarketTimestampSet pullMarketOrderBooksMarkets(TimeWindow timeWindow) override;
+
+ MarketTimestampSet pullTradeMarkets(TimeWindow timeWindow) override;
+
+ MarketOrderBookVector pullMarketOrderBooks(Market market, TimeWindow timeWindow) override;
+
+ PublicTradeVector pullTrades(Market market, TimeWindow timeWindow) override;
+
+ private:
+ ProtobufObjectsDeserializer<::proto::MarketOrderBook, MarketOrderBookConverter> _marketOrderBookDeserializer;
+ ProtobufObjectsDeserializer<::proto::PublicTrade, PublicTradeConverter> _publicTradeDeserializer;
+};
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/include/proto-market-data-serializer.hpp b/src/serialization/include/proto-market-data-serializer.hpp
new file mode 100644
index 00000000..5305709c
--- /dev/null
+++ b/src/serialization/include/proto-market-data-serializer.hpp
@@ -0,0 +1,35 @@
+#pragma once
+
+#include
+#include
+
+#include "abstract-market-data-serializer.hpp"
+#include "market-order-book.pb.h"
+#include "market-timestamp-set.hpp"
+#include "market.hpp"
+#include "proto-public-trade-compare.hpp"
+#include "proto-serializer.hpp"
+#include "public-trade.pb.h"
+#include "publictrade.hpp"
+
+namespace cct {
+
+class MarketOrderBook;
+
+/// This class is responsible of managing the periodic writes to disk of timed market data, for a given exchange.
+/// This class is not thread safe
+class ProtoMarketDataSerializer : public AbstractMarketDataSerializer {
+ public:
+ ProtoMarketDataSerializer(std::string_view dataDir, const MarketTimestampSets &lastWrittenObjectsMarketTimestamp,
+ std::string_view exchangeName);
+
+ void push(const MarketOrderBook &marketOrderBook) override;
+
+ void push(Market market, std::span publicTrades) override;
+
+ private:
+ ProtobufObjectsSerializer<::proto::MarketOrderBook> _marketOrderBookSerializer;
+ ProtobufObjectsSerializer<::proto::PublicTrade, ProtoPublicTradeComp, ProtoPublicTradeEqual> _tradesSerializer;
+};
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/include/proto-market-order-book-converter.hpp b/src/serialization/include/proto-market-order-book-converter.hpp
new file mode 100644
index 00000000..6cff3bb8
--- /dev/null
+++ b/src/serialization/include/proto-market-order-book-converter.hpp
@@ -0,0 +1,21 @@
+#pragma once
+
+#include "market-order-book.pb.h"
+#include "market.hpp"
+#include "marketorderbook.hpp"
+
+namespace cct {
+
+::proto::MarketOrderBook ConvertMarketOrderBookToProto(const MarketOrderBook &marketOrderBook);
+
+class MarketOrderBookConverter {
+ public:
+ explicit MarketOrderBookConverter(Market market) : _market(market) {}
+
+ MarketOrderBook operator()(const ::proto::MarketOrderBook &marketOrderBookTimedData);
+
+ private:
+ Market _market;
+};
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/include/proto-multiple-messages-handler.hpp b/src/serialization/include/proto-multiple-messages-handler.hpp
new file mode 100644
index 00000000..19bb1a0e
--- /dev/null
+++ b/src/serialization/include/proto-multiple-messages-handler.hpp
@@ -0,0 +1,199 @@
+#pragma once
+
+#include
+#include
+#include
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "cct_exception.hpp"
+#include "cct_log.hpp"
+
+namespace cct {
+
+/// Utility class that allows compressed data to be written to rolling files.
+/// At default constructor, no file is opened. Client is expected to call 'open' with a new std::ostream prior to any
+/// write. Data is written with the following scheme:
+/// - First a VarInt64 is written with the size (in bytes) of the object to be serialized.
+/// - Then the object itself is serialized.
+template
+class ProtobufMessagesCompressedWriter {
+ private:
+ static_assert(std::derived_from,
+ "OStreamType for ProtobufMessagesCompressedWriter should derive from std::ostream");
+
+ using OstreamOutputStream = ::google::protobuf::io::OstreamOutputStream;
+ using GzipOutputStream = ::google::protobuf::io::GzipOutputStream;
+ using CodedOutputStream = ::google::protobuf::io::CodedOutputStream;
+
+ public:
+ /// Initializes a new ProtobufMessagesCompressedWriter without any opened stream.
+ /// 'open' method should be called before any write.
+ ProtobufMessagesCompressedWriter() noexcept(std::is_nothrow_default_constructible_v) = default;
+
+ /// No copy / move operations allowed as already deleted by underlying OstreamOutputStream & CodedOutputStream
+ ProtobufMessagesCompressedWriter(const ProtobufMessagesCompressedWriter&) = delete;
+ ProtobufMessagesCompressedWriter(ProtobufMessagesCompressedWriter&&) = delete;
+
+ ProtobufMessagesCompressedWriter& operator=(const ProtobufMessagesCompressedWriter&) = delete;
+ ProtobufMessagesCompressedWriter& operator=(ProtobufMessagesCompressedWriter&&) = delete;
+
+ ~ProtobufMessagesCompressedWriter() { close(); }
+
+ void open(OStreamType&& newOs) {
+ close();
+
+ _os = std::move(newOs);
+
+ OstreamOutputStream* pOos;
+ try {
+ pOos = std::construct_at(getOstreamOutputStreamPtr(), std::addressof(_os));
+ } catch (const std::exception& ex) {
+ _os = OStreamType();
+ throw ex;
+ }
+
+ GzipOutputStream* pGzos;
+ try {
+ pGzos = std::construct_at(getGzipOutputStreamPtr(), pOos);
+ } catch (const std::exception& ex) {
+ std::destroy_at(pOos);
+ _os = OStreamType();
+ throw ex;
+ }
+
+ try {
+ std::construct_at(getCodedOutputStreamPtr(), pGzos);
+ } catch (const std::exception& ex) {
+ std::destroy_at(pGzos);
+ std::destroy_at(pOos);
+ _os = OStreamType();
+ throw ex;
+ }
+
+ _isFileOpened = true;
+ }
+
+ template
+ void write(const MsgT& msg) {
+ if (!_isFileOpened) {
+ throw exception("ProtobufMessagesWriter::open should have been called first");
+ }
+
+ auto* cos = getCodedOutputStreamPtr();
+
+ cos->WriteVarint32(static_cast(msg.ByteSizeLong()));
+
+ if (!msg.SerializeToCodedStream(cos)) {
+ log::error("Failed to serialize to coded stream");
+ }
+ }
+
+ OStreamType flush() noexcept(std::is_nothrow_swappable_v) {
+ close();
+
+ OStreamType ret;
+ ret.swap(_os);
+ return ret;
+ }
+
+ private:
+ void close() noexcept {
+ if (_isFileOpened) {
+ // reverse destroy streams to flush latest data. Recreate the streams after creation of new ofstream
+ std::destroy_at(getCodedOutputStreamPtr());
+ std::destroy_at(getGzipOutputStreamPtr());
+ std::destroy_at(getOstreamOutputStreamPtr());
+ _isFileOpened = false;
+ }
+ }
+
+ OstreamOutputStream* getOstreamOutputStreamPtr() { return reinterpret_cast(_oos); }
+ GzipOutputStream* getGzipOutputStreamPtr() { return reinterpret_cast(_gzos); }
+ CodedOutputStream* getCodedOutputStreamPtr() { return reinterpret_cast(_cos); }
+
+ OStreamType _os;
+ alignas(OstreamOutputStream) std::byte _oos[sizeof(OstreamOutputStream)];
+ alignas(GzipOutputStream) std::byte _gzos[sizeof(GzipOutputStream)];
+ alignas(CodedOutputStream) std::byte _cos[sizeof(CodedOutputStream)];
+ bool _isFileOpened = false;
+};
+
+class ProtobufMessageReaderBase {
+ public:
+ explicit ProtobufMessageReaderBase(google::protobuf::io::ZeroCopyInputStream* is) : _cis(is) {}
+
+ /// Tells whether this reader has at least one more message to be read.
+ bool hasNext() { return _cis.ReadVarintSizeAsInt(&_nextSize); }
+
+ /// Read next message and returns it.
+ /// 'hasNext' should have been called before, and returned true.
+ template
+ MsgT next() {
+ MsgT msg;
+ auto msgLimit = _cis.PushLimit(_nextSize);
+ if (!msg.ParseFromCodedStream(&_cis)) {
+ log::error("Error reading single protobuf message of size {}", _nextSize);
+ }
+ _cis.PopLimit(msgLimit);
+ return msg;
+ }
+
+ private:
+ ::google::protobuf::io::CodedInputStream _cis;
+ int _nextSize{};
+};
+
+/// Uncompressed messages reader iterator, provided as an example. Unused in production code, but can be useful if some
+/// data has been written uncompressed.
+class ProtobufMessageReaderIterator {
+ public:
+ explicit ProtobufMessageReaderIterator(std::istream& is) : _iis(&is), _protobufMessageReaderBase(&_iis) {}
+
+ /// Tells whether this reader has at least one more message to be read.
+ bool hasNext() { return _protobufMessageReaderBase.hasNext(); }
+
+ /// Read next message and returns it.
+ /// 'hasNext' should have been called before, and returned true.
+ template
+ MsgT next() {
+ return _protobufMessageReaderBase.next();
+ }
+
+ private:
+ ::google::protobuf::io::IstreamInputStream _iis;
+ ProtobufMessageReaderBase _protobufMessageReaderBase;
+};
+
+/// The compressed reader iterator that should be used in case files have been written with a
+/// ProtobufMessagesCompressedWriter.
+class ProtobufMessageCompressedReaderIterator {
+ public:
+ explicit ProtobufMessageCompressedReaderIterator(std::istream& is)
+ : _iis(&is), _gzipIs(&_iis), _protobufMessageReaderBase(&_gzipIs) {}
+
+ /// Tells whether this reader has at least one more message to be read.
+ bool hasNext() { return _protobufMessageReaderBase.hasNext(); }
+
+ /// Read next message and returns it.
+ /// 'hasNext' should have been called before, and returned true.
+ template
+ MsgT next() {
+ return _protobufMessageReaderBase.next();
+ }
+
+ private:
+ ::google::protobuf::io::IstreamInputStream _iis;
+ ::google::protobuf::io::GzipInputStream _gzipIs;
+ ProtobufMessageReaderBase _protobufMessageReaderBase;
+};
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/include/proto-public-trade-compare.hpp b/src/serialization/include/proto-public-trade-compare.hpp
new file mode 100644
index 00000000..812a5a1f
--- /dev/null
+++ b/src/serialization/include/proto-public-trade-compare.hpp
@@ -0,0 +1,17 @@
+#pragma once
+
+namespace proto {
+class PublicTrade;
+}
+
+namespace cct {
+
+struct ProtoPublicTradeComp {
+ bool operator()(const ::proto::PublicTrade &lhs, const ::proto::PublicTrade &rhs) const;
+};
+
+struct ProtoPublicTradeEqual {
+ bool operator()(const ::proto::PublicTrade &lhs, const ::proto::PublicTrade &rhs) const;
+};
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/include/proto-public-trade-converter.hpp b/src/serialization/include/proto-public-trade-converter.hpp
new file mode 100644
index 00000000..ad2d7add
--- /dev/null
+++ b/src/serialization/include/proto-public-trade-converter.hpp
@@ -0,0 +1,21 @@
+#pragma once
+
+#include "market.hpp"
+#include "public-trade.pb.h"
+#include "publictrade.hpp"
+
+namespace cct {
+
+::proto::PublicTrade ConvertPublicTradeToProto(const PublicTrade &publicTrade);
+
+class PublicTradeConverter {
+ public:
+ explicit PublicTradeConverter(Market market) : _market(market) {}
+
+ PublicTrade operator()(const ::proto::PublicTrade &protoPublicTrade) const;
+
+ private:
+ Market _market;
+};
+
+} // namespace cct
\ No newline at end of file
diff --git a/src/serialization/include/proto-serializer.hpp b/src/serialization/include/proto-serializer.hpp
new file mode 100644
index 00000000..017643da
--- /dev/null
+++ b/src/serialization/include/proto-serializer.hpp
@@ -0,0 +1,264 @@
+#pragma once
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "cct_exception.hpp"
+#include "cct_log.hpp"
+#include "cct_vector.hpp"
+#include "durationstring.hpp"
+#include "market-timestamp-set.hpp"
+#include "market.hpp"
+#include "proto-multiple-messages-handler.hpp"
+#include "serialization-tools.hpp"
+#include "stringhelpers.hpp"
+#include "timedef.hpp"
+
+namespace cct {
+
+/// Class responsible to accumulate protobuf objects in memory and perform regular flushes of its data to the disk.
+/// Data is accumulated by Market and will write to following files (from subPath):
+/// 'BASECUR-QUOTECUR/YYYY/MM/DD/HH:00:00_HH:59:59.binpb'
+///
+/// If you may push duplicated objects, you have to provide Comp and Equal types.
+/// In this case, Equal must be consistent with Comp, and the first criteria of the comparison should be the timestamp
+/// (ordered from oldest to youngest).
+///
+/// You may not provide any Comp and Equal if by design you will not push duplicated data.
+template
+class ProtobufObjectsSerializer {
+ public:
+ /// Creates a new ProtobufObjectsSerializer.
+ /// @param marketTimestampSet the latest written timestamp for all markets to avoid writing duplicate entries between
+ /// coincenter restarts.
+ ProtobufObjectsSerializer(std::filesystem::path subPath, const MarketTimestampSet &marketTimestampSet,
+ int32_t nbObjectsPerMarketInMemory)
+ : _subPath(std::move(subPath)), _nbObjectsPerMarketInMemory(nbObjectsPerMarketInMemory) {
+ for (const auto &[market, timestamp] : marketTimestampSet) {
+ auto &lastWrittenObjectTimestamp = _marketDataMap[market].lastWrittenObjectTimestamp;
+
+ lastWrittenObjectTimestamp = timestamp;
+
+ // When program starts, we want to exclude equal timestamps to avoid writing objects that may have been written
+ // already from a previous run (the SortUnique will not protect us here)
+ ++lastWrittenObjectTimestamp;
+ }
+ }
+
+ ProtobufObjectsSerializer(const ProtobufObjectsSerializer &) = delete;
+ ProtobufObjectsSerializer &operator=(const ProtobufObjectsSerializer &) = delete;
+
+ ProtobufObjectsSerializer(ProtobufObjectsSerializer &&other) noexcept { swap(other); }
+
+ ProtobufObjectsSerializer &operator=(ProtobufObjectsSerializer &&other) noexcept {
+ if (&other != this) {
+ swap(other);
+ }
+ return *this;
+ }
+
+ /// At destruction of the serializer, we try to write all remaining objects in the buffer (as best effort mode).
+ ~ProtobufObjectsSerializer() {
+ try {
+ for (auto &[market, marketData] : _marketDataMap) {
+ writeOnDisk(market, marketData);
+ }
+ } catch (const std::exception &e) {
+ log::error("exception caught in writeOnDisk at ProtobufObjectsSerializer destruction: {}", e.what());
+ }
+ }
+
+ /// Pushes a new object into the serializer.
+ /// The new object is guaranteed to be written upon destruction of this serializer at the latest unless:
+ /// - its timestamp is older than the latest written timestamp of this market
+ /// - it has invalid data
+ template
+ void push(Market market, ProtobufObjectTypeU &&protoObj) {
+ if (!protoObj.has_unixtimestampinms()) {
+ throw exception("Attempt to push proto object without any timestamp");
+ }
+
+ auto &marketData = _marketDataMap[market];
+ if (TimePoint{milliseconds{protoObj.unixtimestampinms()}} < marketData.lastWrittenObjectTimestamp) {
+ // do not push an object that has an older timestamp of the last written object
+ return;
+ }
+
+ marketData.dataVector.push_back(std::forward(protoObj));
+
+ checkWriteOnDisk(market, marketData);
+ }
+
+ void swap(ProtobufObjectsSerializer &rhs) noexcept {
+ _marketDataMap.swap(rhs._marketDataMap);
+ _subPath.swap(rhs._subPath);
+ std::swap(_nbObjectsPerMarketInMemory, rhs._nbObjectsPerMarketInMemory);
+ std::swap(_flushCounter, rhs._flushCounter);
+ }
+
+ private:
+ using ProtobufObjectTypeVector = vector;
+
+ struct MarketData {
+ ProtobufObjectTypeVector dataVector;
+ TimePoint lastWrittenObjectTimestamp;
+ };
+
+ void checkWriteOnDisk(Market market, MarketData &marketData) {
+ auto &dataVector = marketData.dataVector;
+ if (dataVector.size() == static_cast(_nbObjectsPerMarketInMemory)) {
+ writeOnDisk(market, marketData);
+
+ // shrink_to_fit as vector will never grow-up larger than its current size
+ dataVector.shrink_to_fit();
+ dataVector.clear();
+
+ checkPeriodicFlush();
+ }
+ }
+
+ void writeOnDisk(Market market, MarketData &marketData) {
+ auto &dataVector = marketData.dataVector;
+ if (dataVector.empty()) {
+ return;
+ }
+
+ const auto nowTime = std::chrono::steady_clock::now();
+
+ SortUnique(dataVector);
+
+ std::filesystem::path path;
+
+ std::chrono::hours prevHourOfDay{-1};
+
+ ProtobufMessagesCompressedWriter protobufMessagesWriter;
+
+ for (const auto &protobufObject : dataVector) {
+ checkOpenFile(market, protobufObject, prevHourOfDay, path, protobufMessagesWriter);
+
+ protobufMessagesWriter.write(protobufObject);
+ }
+
+ marketData.lastWrittenObjectTimestamp = TimePoint{milliseconds{dataVector.back().unixtimestampinms()}};
+
+ const auto nbElemsWritten = dataVector.size();
+
+ const auto steadyClockDuration = std::chrono::steady_clock::now() - nowTime;
+ const auto dur = std::chrono::duration_cast(steadyClockDuration);
+
+ log::info("Serialized {} object(s) for {} data in {}, last in {}", nbElemsWritten, market, DurationToString(dur),
+ path.string());
+ }
+
+ // Periodic memory release to avoid possible leaks for long time running (if market data unused anymore for instance)
+ void checkPeriodicFlush() {
+ if (++_flushCounter != RehashThreshold) {
+ return;
+ }
+
+ _flushCounter = 0;
+
+ auto nowTime = Clock::now();
+
+ for (auto it = _marketDataMap.begin(); it != _marketDataMap.end();) {
+ if (it->second.lastWrittenObjectTimestamp + DurationType{static_cast(DurationValue)} < nowTime) {
+ // Unchanged data since a long time - write data if any, and clears the entry in the map
+ const Market market = it->first;
+ MarketData &marketData = it->second;
+
+ writeOnDisk(market, marketData);
+
+ log::info("Released {} protobuf objects for {}", marketData.dataVector.capacity(), market);
+
+ it = _marketDataMap.erase(it);
+ } else {
+ ++it;
+ }
+ }
+
+ _marketDataMap.rehash(_marketDataMap.size());
+ }
+
+ static void SortUnique(ProtobufObjectTypeVector &dataVector) {
+ static_assert((std::is_void_v && std::is_void_v) || (!std::is_void_v && !std::is_void_v));
+
+ if constexpr (std::is_void_v) {
+ // Sort by timestamp (required by 'writeOnDisk' algorithm)
+ std::ranges::sort(dataVector, [](const auto &lhs, const auto &rhs) {
+ return lhs.unixtimestampinms() < rhs.unixtimestampinms();
+ });
+ } else {
+ // We assume that timestamp is the first sorting criteria
+ std::ranges::sort(dataVector, Comp{});
+ }
+
+ // If duplicate elements are possible, remove them
+ if constexpr (!std::is_void_v) {
+ const auto [eraseIt1, eraseIt2] = std::ranges::unique(dataVector, Equal{});
+
+ dataVector.erase(eraseIt1, eraseIt2);
+ }
+ }
+
+ void checkOpenFile(Market market, const ProtobufObjectType &protobufObject, std::chrono::hours &prevHourOfDay,
+ std::filesystem::path &path,
+ ProtobufMessagesCompressedWriter &protobufMessagesWriter) {
+ const TimePoint tp{milliseconds{protobufObject.unixtimestampinms()}};
+ const auto hourOfDay = GetHourOfDay(tp);
+
+ if (prevHourOfDay != hourOfDay) {
+ // open new outfile
+ setDirectory(market.str(), tp, path);
+
+ std::filesystem::create_directories(path);
+
+ path /= ComputeProtoFileName(std::chrono::duration_cast(hourOfDay).count());
+
+ std::ofstream ofs(path, std::ios_base::out | std::ios::binary | std::ios_base::app);
+
+ if (!ofs.is_open()) {
+ throw exception("Cannot open the ofstream for writing to {}: {} (code {})", path.string(), std::strerror(errno),
+ errno);
+ }
+
+ protobufMessagesWriter.open(std::move(ofs));
+ prevHourOfDay = hourOfDay;
+ }
+ }
+
+ static std::chrono::hours GetHourOfDay(TimePoint tp) {
+ const auto dp = std::chrono::floor(tp);
+
+ return std::chrono::floor(tp - dp);
+ }
+
+ void setDirectory(std::string_view marketStr, TimePoint tp, std::filesystem::path &path) {
+ const auto dp = std::chrono::floor(tp);
+ const std::chrono::year_month_day ymd{dp};
+
+ path = _subPath / marketStr;
+ path /= std::string_view(ToCharVector(static_cast(ymd.year())));
+ path /= MonthStr(static_cast(ymd.month()));
+ path /= DayOfMonthStr(static_cast(ymd.day()));
+ }
+
+ using MarketDataMap = std::unordered_map;
+
+ MarketDataMap _marketDataMap;
+ std::filesystem::path _subPath;
+ int32_t _nbObjectsPerMarketInMemory;
+ int32_t _flushCounter{};
+};
+
+} // namespace cct
diff --git a/src/serialization/include/serialization-tools.hpp b/src/serialization/include/serialization-tools.hpp
new file mode 100644
index 00000000..132cff73
--- /dev/null
+++ b/src/serialization/include/serialization-tools.hpp
@@ -0,0 +1,29 @@
+#pragma once
+
+#include
+#include