diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index f13502bf..47f91ede 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -22,7 +22,10 @@ jobs: xcode-version: latest-stable - name: Install dependencies - run: brew install ninja + run: | + brew update && brew upgrade && brew upgrade --cask && brew cleanup + brew install ninja + echo "CC=$(echo ${{matrix.compiler}} | sed -e 's/^g++/gcc/' | sed 's/+//g')" >> $GITHUB_ENV - name: Configure run: cmake -DCCT_ENABLE_ASAN=OFF -S . -B build -GNinja diff --git a/.github/workflows/ubuntu-clang-tidy.yml b/.github/workflows/ubuntu-clang-tidy.yml index 240681e4..f2d2fbf5 100644 --- a/.github/workflows/ubuntu-clang-tidy.yml +++ b/.github/workflows/ubuntu-clang-tidy.yml @@ -37,6 +37,7 @@ jobs: clang-tidy --dump-config cmake -S . -B build -DCCT_ENABLE_CLANG_TIDY=ON -DCCT_ENABLE_ASAN=OFF -GNinja env: + CC: clang-${{matrix.clang-version}} CXX: clang++-${{matrix.clang-version}} CMAKE_BUILD_TYPE: ${{matrix.buildmode}} diff --git a/.github/workflows/ubuntu-monitoring.yml b/.github/workflows/ubuntu-special.yml similarity index 71% rename from .github/workflows/ubuntu-monitoring.yml rename to .github/workflows/ubuntu-special.yml index d91c0c42..de09863a 100644 --- a/.github/workflows/ubuntu-monitoring.yml +++ b/.github/workflows/ubuntu-special.yml @@ -1,4 +1,4 @@ -name: Monitoring +name: Special on: push: @@ -7,14 +7,15 @@ on: pull_request: jobs: - ubuntu-monitoring-build: - name: Build on Ubuntu with monitoring support + ubuntu-special-build: + name: Build on Ubuntu with monitoring / protobuf support runs-on: ubuntu-latest strategy: matrix: compiler: [g++-11] buildmode: [Debug] - build-prometheus-from-source: [0, 1] + build-special-from-source: [0, 1] + prometheus-options: ["-DBUILD_SHARED_LIBS=ON -DENABLE_PULL=OFF -DENABLE_PUSH=ON -DENABLE_COMPRESSION=OFF -DENABLE_TESTING=OFF"] steps: - name: Checkout repository code @@ -24,6 +25,7 @@ jobs: run: | sudo apt update sudo apt install cmake libssl-dev git libcurl4-openssl-dev ninja-build -y --no-install-recommends + echo "CC=$(echo ${{matrix.compiler}} | sed -e 's/^g++/gcc/' | sed 's/+//g')" >> $GITHUB_ENV - name: Install prometheus-cpp run: | @@ -39,10 +41,10 @@ jobs: env: CXX: ${{matrix.compiler}} CMAKE_BUILD_TYPE: ${{matrix.buildmode}} - if: matrix.build-prometheus-from-source == 0 + if: matrix.build-special-from-source == 0 - name: Configure CMake - run: cmake -S . -B build -DCCT_BUILD_PROMETHEUS_FROM_SRC=${{matrix.build-prometheus-from-source}} -DCCT_ENABLE_ASAN=OFF -GNinja + run: cmake -S . -B build -DCCT_BUILD_PROMETHEUS_FROM_SRC=${{matrix.build-special-from-source}} -DCCT_ENABLE_PROTO=${{matrix.build-special-from-source}} -DCCT_ENABLE_ASAN=OFF -GNinja env: CXX: ${{matrix.compiler}} CMAKE_BUILD_TYPE: ${{matrix.buildmode}} @@ -56,4 +58,4 @@ jobs: - name: Sanity check main executable working-directory: ${{github.workspace}}/build - run: ./coincenter --help \ No newline at end of file + run: ./coincenter --help diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index a272d084..4847b203 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -23,6 +23,7 @@ jobs: run: | sudo apt update sudo apt install cmake libssl-dev libcurl4-openssl-dev ninja-build -y --no-install-recommends + echo "CC=$(echo ${{matrix.compiler}} | sed -e 's/^g++/gcc/' | sed 's/+//g')" >> $GITHUB_ENV - name: Install gcc run: | @@ -30,7 +31,7 @@ jobs: # Temporary workaround for libasan bug stated here: https://github.com/google/sanitizers/issues/1716 sudo sysctl vm.mmap_rnd_bits=28 - if: startsWith(matrix.compiler, 'g++') + if: startsWith(matrix.compiler, 'g') - name: Install clang run: | @@ -38,7 +39,7 @@ jobs: wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh sudo ./llvm.sh ${CLANG_VERSION} - if: startsWith(matrix.compiler, 'clang++') + if: startsWith(matrix.compiler, 'clang') - name: Configure CMake run: cmake -S . -B build -GNinja @@ -55,4 +56,4 @@ jobs: - name: Sanity check main executable working-directory: ${{github.workspace}}/build - run: ./coincenter --help \ No newline at end of file + run: ./coincenter --help diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index f579676c..e733879d 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -33,4 +33,4 @@ jobs: - name: Sanity check main executable working-directory: ${{github.workspace}}/build/${{matrix.buildmode}} - run: .\coincenter.exe --help \ No newline at end of file + run: .\coincenter.exe --help diff --git a/.gitignore b/.gitignore index 9ad9a59a..44d7947c 100644 --- a/.gitignore +++ b/.gitignore @@ -64,6 +64,7 @@ data/cache data/log data/secret !data/secret/secret_test.json +data/serialized data/static/exchangeconfig.json data/static/generalconfig.json monitoring/data/grafana/* diff --git a/CMakeLists.txt b/CMakeLists.txt index 4d6f5bbe..a6fd3393 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,6 +38,7 @@ option(CCT_ENABLE_TESTS "Build the unit tests" ${MAIN_PROJECT}) option(CCT_BUILD_EXEC "Build an executable instead of a static library" ${MAIN_PROJECT}) option(CCT_ENABLE_ASAN "Compile with AddressSanitizer" ${CCT_ASAN_BUILD}) option(CCT_ENABLE_CLANG_TIDY "Compile with clang-tidy checks" OFF) +option(CCT_ENABLE_PROTO "Compile with protobuf support (to export data to the outside world)" ON) option(CCT_BUILD_PROMETHEUS_FROM_SRC "Fetch and build from prometheus-cpp sources" OFF) set(CCT_DATA_DIR "${CMAKE_CURRENT_SOURCE_DIR}/data" CACHE PATH "Needed data directory for coincenter. Can also be overridden at runtime with this environment variable") @@ -162,14 +163,68 @@ set(JWT_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) list(APPEND fetchContentPackagesToMakeAvailable jwt-cpp) +# protobuf - serialization / deserialization library +set(PROTOBUF_FETCHED_CONTENT OFF) +if(CCT_ENABLE_PROTO) + find_package(Protobuf CONFIG) + if(Protobuf_FOUND) + message(STATUS "Linking with protobuf ${protobuf_VERSION}") + else() + # Check here for a new version: https://protobuf.dev/support/version-support/#cpp + if (NOT PROTOBUF_VERSION) + set(PROTOBUF_VERSION v5.26.1) + endif() + + message(STATUS "Configuring protobuf ${PROTOBUF_VERSION} from sources") + + # Using git here to simplify cmake code instead of archive download as + # it depends on Abseil behind the scene whose code is not included in the release archives. + FetchContent_Declare( + protobuf + GIT_REPOSITORY https://github.com/protocolbuffers/protobuf.git + GIT_TAG ${PROTOBUF_VERSION} + GIT_SHALLOW true + ) + + set(protobuf_BUILD_TESTS OFF CACHE BOOL "" FORCE) + set(protobuf_BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) + set(protobuf_BUILD_LIBUPB OFF CACHE BOOL "" FORCE) + + # Abseil options + set(ABSL_CXX_STANDARD ${CMAKE_CXX_STANDARD} CACHE STRING "" FORCE) + set(ABSL_PROPAGATE_CXX_STD ON CACHE BOOL "" FORCE) + set(ABSL_ENABLE_INSTALL OFF CACHE BOOL "" FORCE) + set(ABSL_BUILD_TESTING OFF CACHE BOOL "" FORCE) + + if(CCACHE_PROGRAM) + set(protobuf_ALLOW_CCACHE ON CACHE BOOL "" FORCE) + endif() + + set(protobuf_INSTALL OFF CACHE BOOL "" FORCE) + set(protobuf_BUILD_LIBUPB OFF CACHE BOOL "" FORCE) + set(protobuf_WITH_ZLIB ON CACHE BOOL "" FORCE) + SET(protobuf_MSVC_STATIC_RUNTIME OFF CACHE INTERNAL "") + + + list(APPEND fetchContentPackagesToMakeAvailable protobuf) + + set(PROTOBUF_FETCHED_CONTENT ON) + endif() +endif() + # Make fetch content available if(fetchContentPackagesToMakeAvailable) message(STATUS "Configuring packages ${fetchContentPackagesToMakeAvailable}") + FetchContent_MakeAvailable("${fetchContentPackagesToMakeAvailable}") + + if(PROTOBUF_FETCHED_CONTENT) + include(${protobuf_SOURCE_DIR}/cmake/protobuf-generate.cmake) + endif() endif() # Unit Tests -include(cmake/AddUnitTest.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/cmake/AddUnitTest.cmake) # Compiler warnings (only in Debug mode) if(CMAKE_BUILD_TYPE STREQUAL "Debug") @@ -215,13 +270,20 @@ if(CCT_ENABLE_PROMETHEUS) add_compile_definitions(CCT_ENABLE_PROMETHEUS) endif() +if(CCT_ENABLE_PROTO) + add_compile_definitions(CCT_ENABLE_PROTO) + add_compile_definitions("CCT_PROTOBUF_VERSION=\"${PROTOBUF_VERSION}\"") +endif() + # Link to sub folders CMakeLists.txt, from the lowest level to the highest level for documentation # (beware of cyclic dependencies) add_subdirectory(src/tech) add_subdirectory(src/monitoring) add_subdirectory(src/http-request) add_subdirectory(src/objects) +add_subdirectory(src/serialization) add_subdirectory(src/api-objects) +add_subdirectory(src/trading) add_subdirectory(src/api) add_subdirectory(src/engine) add_subdirectory(src/main) diff --git a/CONFIG.md b/CONFIG.md index 9b630a7a..b4222176 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -48,18 +48,19 @@ Configures the logging, tracking activity of relevant commands, and console outp #### General options description -| Name | Value | Description | -| ---------------------------------------------- | ---------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **apiOutputType** | String among {`off`, `table`, `json`} | Configure the default output type of coincenter (can be overridden by command line)queries | -| **fiatConversion.rate** | Duration string (ex: `8h`) | Minimum duration between two consecutive requests of the same fiat conversion | -| **log.activityTracking.commandTypes** | Array of strings (ex: `["Buy", "Sell"]`) | Array of command types whose output will be stored to activity history files. | -| **log.activityTracking.dateFileNameFormat** | String (ex: `%Y-%m` for month split) | Defines the date string format suffix used by activity history files. The string should be compatible with [std::strftime](https://en.cppreference.com/w/cpp/chrono/c/strftime). Old data will never be clean-up by `coincenter` (as it may contain important data). User should manage the clean-up / storage. | -| **log.activityTracking.withSimulatedCommands** | Boolean | When some commands are launched in simulated mode (trades, withdraw for instance), they will be logged if `true`. | -| **log.consoleLevel** | String | Defines the log level for standard output. Can be {'off', 'critical', 'error', 'warning', 'info', 'debug', 'trace'} | -| **log.fileLevel** | String | Defines the log level in files. Can be {'off', 'critical', 'error', 'warning', 'info', 'debug', 'trace'} | -| **log.maxFileSize** | String (ex: `5Mi` for 5 Megabytes) | Defines in bytes the maximum logging file size. A string representation of an integral, possibly with one suffix ending such as k, M, G, T (1k multipliers) or Ki, Mi, Gi, Ti (1024 multipliers) are supported. | -| **log.maxNbFiles** | Integer | Number of maximum rotating files for log in files | -| **requests.concurrency.nbMaxParallelRequests** | Integer | Size of the thread pool that makes exchange requests. | +| Name | Value | Description | +| -------------------------------------------------------- | ---------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **apiOutputType** | String among {`off`, `table`, `json`} | Configure the default output type of coincenter (can be overridden by command line)queries | +| **fiatConversion.rate** | Duration string (ex: `8h`) | Minimum duration between two consecutive requests of the same fiat conversion | +| **log.activityTracking.commandTypes** | Array of strings (ex: `["Buy", "Sell"]`) | Array of command types whose output will be stored to activity history files. | +| **log.activityTracking.dateFileNameFormat** | String (ex: `%Y-%m` for month split) | Defines the date string format suffix used by activity history files. The string should be compatible with [std::strftime](https://en.cppreference.com/w/cpp/chrono/c/strftime). Old data will never be clean-up by `coincenter` (as it may contain important data). User should manage the clean-up / storage. | +| **log.activityTracking.withSimulatedCommands** | Boolean | When some commands are launched in simulated mode (trades, withdraw for instance), they will be logged if `true`. | +| **log.consoleLevel** | String | Defines the log level for standard output. Can be {'off', 'critical', 'error', 'warning', 'info', 'debug', 'trace'} | +| **log.fileLevel** | String | Defines the log level in files. Can be {'off', 'critical', 'error', 'warning', 'info', 'debug', 'trace'} | +| **log.maxFileSize** | String (ex: `5Mi` for 5 Megabytes) | Defines in bytes the maximum logging file size. A string representation of an integral, possibly with one suffix ending such as k, M, G, T (1k multipliers) or Ki, Mi, Gi, Ti (1024 multipliers) are supported. | +| **log.maxNbFiles** | Integer | Number of maximum rotating files for log in files | +| **requests.concurrency.nbMaxParallelRequests** | Integer | Size of the thread pool that makes exchange requests. | +| **trading.automation.deserialization.loadChunkDuration** | Duration string (ex: `1w`) | Time window duration of historic stored data loaded and replayed at once given to the trading engine | ### static/exchangeconfig.json @@ -152,6 +153,7 @@ Refer to the hardcoded default json example as a model in case of doubt. | *query* | **updateFrequency.depositWallet** | Duration string (ex: `1min`) | Minimum duration between two consecutive requests of deposit information (including wallet) | | *query* | **updateFrequency.currencyInfo** | Duration string (ex: `4h`) | Minimum duration between two consecutive requests of dynamic currency info retrieval on Bithumb only (used for place order) | | *query* | **placeSimulateRealOrder** | Boolean (`true` or `false`) | If `true`, in trade simulation mode (with `--sim`) exchanges which do not support simulated mode in place order will actually place a real order, with the following characteristics: This will allow place of a 'real' order that cannot be matched in practice (if it is, lucky you!) | +| *query* | **marketDataSerialization** | Boolean (`true` or `false`) | If `true` and `coincenter` is compiled with **protobuf** support, some market data will automatically be exported in the `data/serialization` directory (`orderbook` and `last-trades`) for a long term storage | | *query* | **multiTradeAllowedByDefault** | Boolean (`true` or `false`) | If `true`, [multi-trade](README.md#multi-trade) will be allowed by default for `trade`, `buy` and `sell`. It can be overridden at command line level with `--no-multi-trade` and `--multi-trade`. | | *query* | **validateApiKey** | Boolean (`true` or `false`) | If `true`, each loaded private key will be tested at start of the program. In case of a failure, it will be removed from the list of private accounts loaded by `coincenter`, so that later queries do not consider it instead of raising a runtime exception. The downside is that it will make an additional check that will make startup slower. | | | *tradefees* | **maker** | String as decimal number representing a percentage (for instance, "0.15") | Trade fees occurring when a maker order is matched | diff --git a/Dockerfile b/Dockerfile index ec800a9f..bb189794 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,11 +5,12 @@ ARG BUILD_MODE=Release ARG BUILD_TEST=0 ARG BUILD_ASAN=0 ARG BUILD_WITH_PROMETHEUS=1 +ARG BUILD_WITH_PROTOBUF=1 # Install base & build dependencies, needed certificates for curl to work with https RUN apt update && \ apt upgrade -y && \ - apt install build-essential ninja-build libssl-dev zlib1g-dev libcurl4-openssl-dev cmake ca-certificates -y --no-install-recommends + apt install build-essential ninja-build libssl-dev zlib1g-dev libcurl4-openssl-dev cmake git ca-certificates -y --no-install-recommends # Copy source files WORKDIR /app/src @@ -41,6 +42,7 @@ RUN cmake -DCMAKE_BUILD_TYPE=${BUILD_MODE} \ -DCCT_ENABLE_TESTS=${BUILD_TEST} \ -DCCT_ENABLE_ASAN=${BUILD_ASAN} \ -DCCT_BUILD_PROMETHEUS_FROM_SRC=${BUILD_WITH_PROMETHEUS} \ + -DCCT_ENABLE_PROTO=${BUILD_WITH_PROTOBUF} \ -GNinja .. # Build diff --git a/INSTALL.md b/INSTALL.md index db8df724..dfe33fc2 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -158,6 +158,7 @@ The minimum tested version is cmake `3.15`, but it's recommended that you use th | `CCT_BUILD_EXEC` | `ON` if main project | Build an executable instead of a static library | | `CCT_ENABLE_ASAN` | `ON` if Debug mode | Compile with AddressSanitizer | | `CCT_ENABLE_CLANG_TIDY` | `ON` if Debug mode and `clang-tidy` is found in `PATH` | Compile with clang-tidy checks | +| `CCT_ENABLE_PROTO` | `ON` | Compile with protobuf support | Example on Linux: to compile it in `Release` mode and `ninja` generator diff --git a/README.md b/README.md index 0169bbff..110058b7 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,16 @@ Main features: - Cancel opened orders - Withdraw (with check at destination that funds are well received) - Dust sweeper - + +## Market data storage & replay + +`coincenter` is able to store the following market data in serialized [binary protobuf](https://protobuf.dev/) format for offline trading algorithm replay: + +- Market order book +- Public trades + +Refer to the dedicated [documentation page](TRADING.md) for more information. + ## Supported exchanges | Exchange | Link | @@ -56,6 +65,7 @@ Main features: - [coincenter](#coincenter) - [Market Data](#market-data) - [Account requests](#account-requests) + - [Market data storage \& replay](#market-data-storage--replay) - [Supported exchanges](#supported-exchanges) - [About](#about) - [Installation](#installation) @@ -1298,4 +1308,4 @@ Possible output: | kucoin | 6090943.32410022531 SHIB | | upbit | 6084383.631243834 SHIB | +----------+------------------------------+ -``` \ No newline at end of file +``` diff --git a/TRADING.md b/TRADING.md new file mode 100644 index 00000000..0e45c519 --- /dev/null +++ b/TRADING.md @@ -0,0 +1,110 @@ +# Trading + +`coincenter` is able to serialize market data and deserialize it later on for future usage. + +## Overview + +Currently, these two sources of data are serializable into [protobuf](https://protobuf.dev/) objects: + +- Market order books +- Public trades + +## Configuration + +### Compilation + +By default, `coincenter` will be built with **protobuf** support (controlled by `cmake` flag `CCT_ENABLE_PROTO` that defaults to `ON`). + +It will try to link to a known installation of **protobuf** if found on current system (oldest tested version is `v25`, make sure to use this version at least), otherwise it will download and compile it from sources. + +### Serialization configuration + +To be able to serialize market data on disk, make sure that you set the **marketDataSerialization** variable to `true` in `exchangeconfig.json` for the exchanges you would like to interact with. +See the [exchange configuration part](CONFIG.md#exchanges-options-description) for more information about how to configure it. + +## Serialization of market data + +The data will be organized by exchange, then market (asset pair, for instance `BTC-USD`), and finally dates (with directories from **year**, **month**, and finally **day** and files as **hours**). + +All will be stored in `coincenter` data directory, under `serialized` sub folder. + +Here is an example of the structure of files you will obtain: + +```bash +data/serialized// +├── binance +│   ├── BTC-EUR +│   │   └── 2024 +│   │   └── 01 +│   │   ├── 11 +│   │   │   └── 22-00-00_22-59-59.binpb +│   │   ├── 12 +│   │   │   ├── 08-00-00_08-59-59.binpb +│   │   │   ├── 09-00-00_09-59-59.binpb +│   │   │   └── 11-00-00_11-59-59.binpb +│   │   └── 14 +│   │   ├── 07-00-00_07-59-59.binpb +│   │   └── 08-00-00_08-59-59.binpb +│   ├── ETH-USDT +│   │   └── 2024 +│   │   ├── 01 +│   │   │   ├── 09 +│   │   │   │   ├── 08-00-00_08-59-59.binpb +│   │   │   │   ├── 09-00-00_09-59-59.binpb +│   │   │   │   ├── 10-00-00_10-59-59.binpb +│   │   │   │   ├── 11-00-00_11-59-59.binpb +├── huobi +│   ├── ADA-USDT +│   │   └── 2024 +│   │   └── 02 +│   │   └── 10 +│   │   └── 16-00-00_16-59-59.binpb +│   ├── BTC-EUR +│   │   └── 2024 +│   │   └── 01 +│   │   ├── 11 +│   │   │   └── 22-00-00_22-59-59.binpb +.... +``` + +To retrieve market data, it's possible to either use multi-commands with both `orderbook` and `last-trades` commands stacked together, or you can use the more handy `market-data` option that is basically a combination of the two without the output by default (it has been created only for serialization purposes). + +For instance, to retrieve continuously data and serialize them indefinitely, you can use the following command: + +```bash +coincenter -r --repeat-time 2s --log warning \ + market-data btc-eur,binance \ + market-data eth-usdt,kucoin \ + market-data ada-usdt,huobi \ + market-data btc-eur,kraken +``` + +Note the usage of the `-r` (repeat option) to keep querying the data as long as you leave `coincenter` up. It's a good idea to also limit the number of logs with setting console log level to `warning` but not mandatory. + +With this command running for an extended period of time, you should obtain a list of files like in the above example. + +Stacking `market-data` commands together with different exchanges (like in the above example) will allow `coincenter` to perform the queries in parallel, ensuring optimal frequency of data updates. This optimization may be implemented for other commands in the future, but it's currently supported only for `market-data`. + +### Graceful shutdown + +Data is flushed on the disk at regular intervals (around 10 minutes). If you wish to restart / shutdown `coincenter` with an infinite `repeat` command to store continuously market data, you can send `SIGINT` or `SIGTERM` so that `coincenter` can gracefully stop after current request and flush its remaining data on disk before shutdown. + +## Replaying historic market data + +Of course, serialization is useful only if we re-use the data one day. Being **protobuf**, not only `coincenter` could read them, but also other tools, but `coincenter` is also able to read this data. + +Locate the `.proto` files in order and feed them to the external tool in case you want to use another program to read the data. Data is stored in hours in streaming mode (see documentation [here](https://protobuf.dev/programming-guides/techniques/#streaming)), and written in compressed format. + +If you want to use a third-party tool to read this data, locate the `.proto` files in the `src` directory for your external program to be able to deserialize the `.binpb` files. + +We will focus here on `coincenter` features concerning this data. + +### Experimental - Testing trading algorithms + +`coincenter` embeds a trading simulator engine that is able to be used for any custom trading algorithm that would derive from the interface. + +This trading simulator will read chunks of historic data stored in **protobuf** and inject them in trading algorithms. + +Locate the `AbstractMarketTrader` class and derive it - you need to return a `TraderCommand` for each market order book and a list of last public trades that occurred at this specific point of time. + +TODO: extend this documentation. diff --git a/alpine.Dockerfile b/alpine.Dockerfile index bd083fab..cbca0fb8 100644 --- a/alpine.Dockerfile +++ b/alpine.Dockerfile @@ -5,9 +5,10 @@ ARG BUILD_MODE=Release ARG BUILD_TEST=0 ARG BUILD_ASAN=0 ARG BUILD_WITH_PROMETHEUS=1 +ARG BUILD_WITH_PROTOBUF=1 # Install base & build dependencies, needed certificates for curl to work with https -RUN apk add --update --upgrade --no-cache g++ libc-dev openssl-dev zlib-dev curl-dev cmake ninja ca-certificates +RUN apk add --update --upgrade --no-cache linux-headers g++ zlib-dev openssl-dev curl-dev cmake ninja git ca-certificates # Copy source files WORKDIR /app/src @@ -39,6 +40,7 @@ RUN cmake -DCMAKE_BUILD_TYPE=${BUILD_MODE} \ -DCCT_ENABLE_TESTS=${BUILD_TEST} \ -DCCT_ENABLE_ASAN=${BUILD_ASAN} \ -DCCT_BUILD_PROMETHEUS_FROM_SRC=${BUILD_WITH_PROMETHEUS} \ + -DCCT_ENABLE_PROTO=${BUILD_WITH_PROTOBUF} \ -GNinja .. # Build diff --git a/src/api/common/CMakeLists.txt b/src/api/common/CMakeLists.txt index 9464b950..76480e03 100644 --- a/src/api/common/CMakeLists.txt +++ b/src/api/common/CMakeLists.txt @@ -1,12 +1,12 @@ aux_source_directory(src API_COMMON_SRC) -include(FindOpenSSL) - add_library(coincenter_api-common STATIC ${API_COMMON_SRC}) + target_link_libraries(coincenter_api-common PUBLIC coincenter_api-objects) target_link_libraries(coincenter_api-common PUBLIC coincenter_objects) target_link_libraries(coincenter_api-common PUBLIC coincenter_http-request) +target_link_libraries(coincenter_api-common PUBLIC coincenter_serialization) target_link_libraries(coincenter_api-common PRIVATE OpenSSL::SSL) target_include_directories(coincenter_api-common PUBLIC include) diff --git a/src/api/common/include/exchangepublicapi.hpp b/src/api/common/include/exchangepublicapi.hpp index f0e24a5b..586e76ca 100644 --- a/src/api/common/include/exchangepublicapi.hpp +++ b/src/api/common/include/exchangepublicapi.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -10,15 +11,20 @@ #include "currencyexchangeflatset.hpp" #include "exchangebase.hpp" #include "exchangepublicapitypes.hpp" +#include "market-order-book-vector.hpp" +#include "market-timestamp-set.hpp" #include "market.hpp" #include "marketorderbook.hpp" #include "monetaryamount.hpp" #include "monetaryamountbycurrencyset.hpp" #include "priceoptions.hpp" #include "public-trade-vector.hpp" +#include "time-window.hpp" namespace cct { +class AbstractMarketDataDeserializer; +class AbstractMarketDataSerializer; class CoincenterInfo; class ExchangeConfig; class FiatConverter; @@ -40,7 +46,7 @@ class ExchangePublic : public ExchangeBase { kWithPossibleFiatConversionAtExtremity }; - virtual ~ExchangePublic() = default; + virtual ~ExchangePublic(); /// Check if public exchange is responding to basic health check, return true in this case. /// Exchange that implements the HealthCheck do not need to add a retry mechanism. @@ -97,14 +103,14 @@ class ExchangePublic : public ExchangeBase { /// Retrieve the order book of given market. /// It should be more precise that previous version with possibility to go deeper. - virtual MarketOrderBook queryOrderBook(Market mk, int depth = kDefaultDepth) = 0; + MarketOrderBook getOrderBook(Market mk, int depth = kDefaultDepth); + + /// Retrieve an ordered vector of recent last trades + PublicTradeVector getLastTrades(Market mk, int nbTrades = kNbLastTradesDefault); /// Retrieve the total volume exchange on given market in the last 24 hours. virtual MonetaryAmount queryLast24hVolume(Market mk) = 0; - /// Retrieve an ordered vector of recent last trades - virtual PublicTradeVector queryLastTrades(Market mk, int nbTrades = kNbLastTradesDefault) = 0; - /// Retrieve the last price of given market. virtual MonetaryAmount queryLastPrice(Market mk) = 0; @@ -176,7 +182,22 @@ class ExchangePublic : public ExchangeBase { /// If no data found, return a 0 MonetaryAmount on given currency. MonetaryAmount queryWithdrawalFeeOrZero(CurrencyCode currencyCode); + MarketTimestampSet pullMarketOrderBooksMarkets(TimeWindow timeWindow); + + MarketTimestampSet pullTradeMarkets(TimeWindow timeWindow); + + PublicTradeVector pullTradesForReplay(Market market, TimeWindow timeWindow); + + MarketOrderBookVector pullMarketOrderBooksForReplay(Market market, TimeWindow timeWindow); + protected: + /// Retrieve the order book of given market. + /// It should be more precise that previous version with possibility to go deeper. + virtual MarketOrderBook queryOrderBook(Market mk, int depth = kDefaultDepth) = 0; + + /// Retrieve an ordered vector of recent last trades + virtual PublicTradeVector queryLastTrades(Market mk, int nbTrades = kNbLastTradesDefault) = 0; + friend class ExchangePrivate; ExchangePublic(std::string_view name, FiatConverter &fiatConverter, CommonAPI &commonApi, @@ -188,7 +209,12 @@ class ExchangePublic : public ExchangeBase { CommonAPI &_commonApi; const CoincenterInfo &_coincenterInfo; const ExchangeConfig &_exchangeConfig; + std::unique_ptr _marketDataDeserializerPtr; + std::unique_ptr _marketDataSerializerPtr; std::recursive_mutex _publicRequestsMutex; + + private: + AbstractMarketDataSerializer &getMarketDataSerializer(); }; } // namespace api } // namespace cct diff --git a/src/api/common/src/exchangeprivateapi.cpp b/src/api/common/src/exchangeprivateapi.cpp index e7621c9d..79ec508d 100644 --- a/src/api/common/src/exchangeprivateapi.cpp +++ b/src/api/common/src/exchangeprivateapi.cpp @@ -557,7 +557,7 @@ PlaceOrderInfo ExchangePrivate::placeOrderProcess(MonetaryAmount &from, Monetary if (tradeInfo.options.isSimulation() && !isSimulatedOrderSupported()) { if (exchangeConfig().placeSimulateRealOrder()) { log::debug("Place simulate real order - price {} will be overriden", price); - MarketOrderBook marketOrderbook = _exchangePublic.queryOrderBook(mk); + MarketOrderBook marketOrderbook = _exchangePublic.getOrderBook(mk); price = isSell ? marketOrderbook.getHighestTheoreticalPrice() : marketOrderbook.getLowestTheoreticalPrice(); } else { PlaceOrderInfo placeOrderInfo = computeSimulatedMatchedPlacedOrderInfo(volume, price, tradeInfo); diff --git a/src/api/common/src/exchangepublicapi.cpp b/src/api/common/src/exchangepublicapi.cpp index d22015db..a16bddc9 100644 --- a/src/api/common/src/exchangepublicapi.cpp +++ b/src/api/common/src/exchangepublicapi.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -25,21 +26,44 @@ #include "exchangeconfig.hpp" #include "exchangepublicapitypes.hpp" #include "fiatconverter.hpp" +#include "market-timestamp-set.hpp" #include "market.hpp" #include "marketorderbook.hpp" #include "monetaryamount.hpp" #include "priceoptions.hpp" #include "priceoptionsdef.hpp" +#include "time-window.hpp" +#include "timedef.hpp" #include "unreachable.hpp" +#ifdef CCT_ENABLE_PROTO +#include "proto-market-data-deserializer.hpp" +#include "proto-market-data-serializer.hpp" +#else +#include "dummy-market-data-deserializer.hpp" +#include "dummy-market-data-serializer.hpp" +#endif + namespace cct::api { + +#ifdef CCT_ENABLE_PROTO +using MarketDataDeserializer = ProtoMarketDataDeserializer; +using MarketDataSerializer = ProtoMarketDataSerializer; +#else +using MarketDataDeserializer = DummyMarketDataDeserializer; +using MarketDataSerializer = DummyMarketDataSerializer; +#endif + ExchangePublic::ExchangePublic(std::string_view name, FiatConverter &fiatConverter, CommonAPI &commonApi, const CoincenterInfo &coincenterInfo) : _name(name), _fiatConverter(fiatConverter), _commonApi(commonApi), _coincenterInfo(coincenterInfo), - _exchangeConfig(coincenterInfo.exchangeConfig(name)) {} + _exchangeConfig(coincenterInfo.exchangeConfig(name)), + _marketDataDeserializerPtr(new MarketDataDeserializer(coincenterInfo.dataDir(), name)) {} + +ExchangePublic::~ExchangePublic() = default; std::optional ExchangePublic::convert(MonetaryAmount from, CurrencyCode toCurrency, const MarketsPath &conversionPath, const CurrencyCodeSet &fiats, @@ -279,7 +303,7 @@ ExchangePublic::CurrenciesPath ExchangePublic::findCurrenciesPath(CurrencyCode f std::optional ExchangePublic::computeLimitOrderPrice(Market mk, CurrencyCode fromCurrencyCode, const PriceOptions &priceOptions) { const int depth = priceOptions.isRelativePrice() ? std::abs(priceOptions.relativePrice()) : 1; - return queryOrderBook(mk, depth).computeLimitPrice(fromCurrencyCode, priceOptions); + return getOrderBook(mk, depth).computeLimitPrice(fromCurrencyCode, priceOptions); } std::optional ExchangePublic::computeAvgOrderPrice(Market mk, MonetaryAmount from, @@ -293,7 +317,7 @@ std::optional ExchangePublic::computeAvgOrderPrice(Market mk, Mo } else if (priceOptions.priceStrategy() == PriceStrategy::kTaker) { depth = kDefaultDepth; } - return queryOrderBook(mk, depth).computeAvgPrice(from, priceOptions); + return getOrderBook(mk, depth).computeAvgPrice(from, priceOptions); } std::optional ExchangePublic::RetrieveMarket(CurrencyCode c1, CurrencyCode c2, const MarketSet &markets) { @@ -434,4 +458,62 @@ MonetaryAmount ExchangePublic::queryWithdrawalFeeOrZero(CurrencyCode currencyCod return withdrawFee; } +MarketOrderBook ExchangePublic::getOrderBook(Market mk, int depth) { + std::lock_guard guard(_publicRequestsMutex); + const auto marketOrderBook = queryOrderBook(mk, depth); + + if (_exchangeConfig.withMarketDataSerialization()) { + getMarketDataSerializer().push(marketOrderBook); + } + return marketOrderBook; +} + +/// Retrieve an ordered vector of recent last trades +PublicTradeVector ExchangePublic::getLastTrades(Market mk, int nbTrades) { + std::lock_guard guard(_publicRequestsMutex); + const auto lastTrades = queryLastTrades(mk, nbTrades); + + if (_exchangeConfig.withMarketDataSerialization()) { + getMarketDataSerializer().push(mk, lastTrades); + } + return lastTrades; +} + +MarketTimestampSet ExchangePublic::pullMarketOrderBooksMarkets(TimeWindow timeWindow) { + return _marketDataDeserializerPtr->pullMarketOrderBooksMarkets(timeWindow); +} + +MarketTimestampSet ExchangePublic::pullTradeMarkets(TimeWindow timeWindow) { + return _marketDataDeserializerPtr->pullTradeMarkets(timeWindow); +} + +PublicTradeVector ExchangePublic::pullTradesForReplay(Market market, TimeWindow timeWindow) { + return _marketDataDeserializerPtr->pullTrades(market, timeWindow); +} + +MarketOrderBookVector ExchangePublic::pullMarketOrderBooksForReplay(Market market, TimeWindow timeWindow) { + return _marketDataDeserializerPtr->pullMarketOrderBooks(market, timeWindow); +} + +AbstractMarketDataSerializer &ExchangePublic::getMarketDataSerializer() { + if (_marketDataSerializerPtr) { + return *_marketDataSerializerPtr; + } + + const auto nowTime = Clock::now(); + + // Heuristic: load up to 1 week of data to retrieve the youngest written timestamp. + // This will be used in order not to write duplicate objects at the start of a new program after that a previous + // program run was stopped. + const TimeWindow largeTimeWindow{nowTime - std::chrono::weeks{1}, nowTime}; + + const MarketTimestampSets marketTimestampSets{pullMarketOrderBooksMarkets(largeTimeWindow), + pullTradeMarkets(largeTimeWindow)}; + + _marketDataSerializerPtr = + std::make_unique(_coincenterInfo.dataDir(), marketTimestampSets, name()); + + return *_marketDataSerializerPtr; +} + } // namespace cct::api diff --git a/src/api/exchanges/src/binancepublicapi.cpp b/src/api/exchanges/src/binancepublicapi.cpp index 12516dae..c4ede616 100644 --- a/src/api/exchanges/src/binancepublicapi.cpp +++ b/src/api/exchanges/src/binancepublicapi.cpp @@ -238,7 +238,7 @@ MonetaryAmount BinancePublic::sanitizePrice(Market mk, MonetaryAmount pri) { MonetaryAmount BinancePublic::computePriceForNotional(Market mk, int avgPriceMins) { if (avgPriceMins == 0) { // price should be the last matched price - PublicTradeVector lastTrades = queryLastTrades(mk, 1); + PublicTradeVector lastTrades = getLastTrades(mk, 1); if (!lastTrades.empty()) { return lastTrades.front().price(); } diff --git a/src/api/exchanges/src/bithumbpublicapi.cpp b/src/api/exchanges/src/bithumbpublicapi.cpp index ed21261f..1779a42c 100644 --- a/src/api/exchanges/src/bithumbpublicapi.cpp +++ b/src/api/exchanges/src/bithumbpublicapi.cpp @@ -136,7 +136,7 @@ std::optional BithumbPublic::queryWithdrawalFee(CurrencyCode cur MonetaryAmount BithumbPublic::queryLastPrice(Market mk) { // Bithumb does not have a REST API endpoint for last price, let's compute it from the orderbook - std::optional avgPrice = queryOrderBook(mk).averagePrice(); + std::optional avgPrice = getOrderBook(mk).averagePrice(); if (!avgPrice) { log::error("Empty order book for {} on {} cannot compute average price", mk, _name); return MonetaryAmount(0, mk.quote()); diff --git a/src/api/interface/include/exchange.hpp b/src/api/interface/include/exchange.hpp index a542430f..f6a5b12e 100644 --- a/src/api/interface/include/exchange.hpp +++ b/src/api/interface/include/exchange.hpp @@ -94,12 +94,12 @@ class Exchange { return apiPublic().queryAllApproximatedOrderBooks(depth); } - MarketOrderBook queryOrderBook(Market mk, int depth = ExchangePublic::kDefaultDepth); + MarketOrderBook getOrderBook(Market mk, int depth = ExchangePublic::kDefaultDepth); MonetaryAmount queryLast24hVolume(Market mk) { return apiPublic().queryLast24hVolume(mk); } /// Retrieve an ordered vector of recent last trades - PublicTradeVector queryLastTrades(Market mk, int nbTrades = ExchangePublic::kNbLastTradesDefault); + PublicTradeVector getLastTrades(Market mk, int nbTrades = ExchangePublic::kNbLastTradesDefault); /// Retrieve the last price of given market. MonetaryAmount queryLastPrice(Market mk) { return apiPublic().queryLastPrice(mk); } diff --git a/src/api/interface/src/exchange.cpp b/src/api/interface/src/exchange.cpp index 1763ae7e..f33f7a4c 100644 --- a/src/api/interface/src/exchange.cpp +++ b/src/api/interface/src/exchange.cpp @@ -48,12 +48,10 @@ bool Exchange::canDeposit(CurrencyCode currencyCode, const CurrencyExchangeFlatS return lb->canDeposit(); } -MarketOrderBook Exchange::queryOrderBook(Market mk, int depth) { return apiPublic().queryOrderBook(mk, depth); } +MarketOrderBook Exchange::getOrderBook(Market mk, int depth) { return apiPublic().getOrderBook(mk, depth); } /// Retrieve an ordered vector of recent last trades -PublicTradeVector Exchange::queryLastTrades(Market mk, int nbTrades) { - return apiPublic().queryLastTrades(mk, nbTrades); -} +PublicTradeVector Exchange::getLastTrades(Market mk, int nbTrades) { return apiPublic().getLastTrades(mk, nbTrades); } void Exchange::updateCacheFile() const { apiPublic().updateCacheFile(); diff --git a/src/engine/CMakeLists.txt b/src/engine/CMakeLists.txt index f6191775..791f7987 100644 --- a/src/engine/CMakeLists.txt +++ b/src/engine/CMakeLists.txt @@ -6,6 +6,8 @@ target_link_libraries(coincenter_engine PUBLIC coincenter_api-common) target_link_libraries(coincenter_engine PUBLIC coincenter_api-exchange) target_link_libraries(coincenter_engine PUBLIC coincenter_api-interface) target_link_libraries(coincenter_engine PUBLIC coincenter_objects) +target_link_libraries(coincenter_engine PUBLIC coincenter_trading-algorithms) +target_link_libraries(coincenter_engine PUBLIC coincenter_trading-common) target_include_directories(coincenter_engine PUBLIC include) add_unit_test( @@ -81,6 +83,13 @@ add_unit_test( ../api/common/test/include ) +add_unit_test( + replay-algorithm-name-iterator_test + test/replay-algorithm-name-iterator_test.cpp + LIBRARIES + coincenter_engine +) + add_unit_test( stringoptionparser_test test/stringoptionparser_test.cpp diff --git a/src/engine/include/coincenter-commands-iterator.hpp b/src/engine/include/coincenter-commands-iterator.hpp new file mode 100644 index 00000000..809037dd --- /dev/null +++ b/src/engine/include/coincenter-commands-iterator.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include + +#include "coincentercommand.hpp" + +namespace cct { + +class CoincenterCommandsIterator { + public: + using CoincenterCommandSpan = std::span; + + /// Initializes a new iterator on all coincenter commands. + explicit CoincenterCommandsIterator(CoincenterCommandSpan commands = CoincenterCommandSpan()) noexcept; + + /// Returns 'true' if this iterator has still some command groups. + bool hasNextCommandGroup() const; + + /// Get next grouped commands and advance the iterator. + /// The grouped commands are guaranteed to have same type and make it possible to parallelize requests when possible. + CoincenterCommandSpan nextCommandGroup(); + + private: + CoincenterCommandSpan _commands; + CoincenterCommandSpan::size_type _pos; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/engine/include/coincenter.hpp b/src/engine/include/coincenter.hpp index eecef086..3cc68998 100644 --- a/src/engine/include/coincenter.hpp +++ b/src/engine/include/coincenter.hpp @@ -4,6 +4,8 @@ #include #include "apikeysprovider.hpp" +#include "cct_const.hpp" +#include "cct_fixedcapacityvector.hpp" #include "coincenterinfo.hpp" #include "commonapi.hpp" #include "exchange-names.hpp" @@ -11,14 +13,18 @@ #include "exchangepool.hpp" #include "exchangesorchestrator.hpp" #include "fiatconverter.hpp" +#include "market-trader-engine.hpp" +#include "market.hpp" #include "metricsexporter.hpp" #include "ordersconstraints.hpp" #include "queryresultprinter.hpp" #include "queryresulttypes.hpp" +#include "replay-options.hpp" #include "transferablecommandresult.hpp" namespace cct { +class AbstractMarketTraderFactory; class CoincenterCommand; class CoincenterCommands; class TradeOptions; @@ -30,6 +36,7 @@ class Coincenter { Coincenter(const CoincenterInfo &coincenterInfo, const ExchangeSecretsInfo &exchangeSecretsInfo); + /// Launch given commands and return the number of processed commands. int process(const CoincenterCommands &coincenterCommands); ExchangeHealthCheckStatus healthCheck(ExchangeNameSpan exchangeNames); @@ -50,6 +57,10 @@ class Coincenter { CurrencyCode equiCurrencyCode, std::optional depth = std::nullopt); + /// Query market data without returning it. + /// This method is especially useful for serialization and metric exports. + void queryMarketDataPerExchange(std::span marketPerPublicExchange); + /// Retrieve the last 24h traded volume for exchanges supporting given market. MonetaryAmountPerExchange getLast24hTradedVolumePerExchange(Market mk, ExchangeNameSpan exchangeNames); @@ -132,6 +143,16 @@ class Coincenter { const ExchangeName &toPrivateExchangeName, const WithdrawOptions &withdrawOptions); + /// Retrieves the markets available for replay for exchanges selection that has some data during the last + /// 'replayDuration' time (so within the time frame [now - replayDuration, now]) + MarketTimestampSetsPerExchange getMarketsAvailableForReplay(const ReplayOptions &replayOptions, + ExchangeNameSpan exchangeNames); + + /// Replay all markets for exchanges selection that has some data during the last + /// 'replayDuration' time (so within the time frame [now - replayDuration, now]) + void replay(const AbstractMarketTraderFactory &marketTraderFactory, const ReplayOptions &replayOptions, Market market, + ExchangeNameSpan exchangeNames); + /// Dumps the content of all file caches in data directory to save cURL queries. void updateFileCaches() const; @@ -147,8 +168,23 @@ class Coincenter { const FiatConverter &fiatConverter() const { return _fiatConverter; } private: - TransferableCommandResultVector processCommand( - const CoincenterCommand &cmd, std::span previousTransferableResults); + TransferableCommandResultVector processGroupedCommands( + std::span groupedCommands, + std::span previousTransferableResults); + + using MarketTraderEngineVector = FixedCapacityVector; + + void replayAlgorithm(const AbstractMarketTraderFactory &marketTraderFactory, std::string_view algorithmName, + const ReplayOptions &replayOptions, std::span marketTraderEngines, + const PublicExchangeNameVector &exchangesWithThisMarketData); + + // TODO: may be moved somewhere else? + MarketTraderEngineVector createMarketTraderEngines(const ReplayOptions &replayOptions, Market market, + PublicExchangeNameVector &exchangesWithThisMarketData); + + MarketTradeRangeStatsPerExchange tradingProcess(const ReplayOptions &replayOptions, + std::span marketTraderEngines, + ExchangeNameSpan exchangesWithThisMarketData); const CoincenterInfo &_coincenterInfo; api::CommonAPI _commonAPI; diff --git a/src/engine/include/coincentercommand.hpp b/src/engine/include/coincentercommand.hpp index df683d1f..db0894d1 100644 --- a/src/engine/include/coincentercommand.hpp +++ b/src/engine/include/coincentercommand.hpp @@ -13,6 +13,7 @@ #include "market.hpp" #include "monetaryamount.hpp" #include "ordersconstraints.hpp" +#include "replay-options.hpp" #include "tradeoptions.hpp" #include "withdrawoptions.hpp" #include "withdrawsconstraints.hpp" @@ -44,6 +45,8 @@ class CoincenterCommand { CoincenterCommand& setCur1(CurrencyCode cur1); CoincenterCommand& setCur2(CurrencyCode cur2); + CoincenterCommand& setReplayOptions(ReplayOptions replayOptions); + CoincenterCommand& setPercentageAmount(bool value = true); CoincenterCommand& withBalanceInUse(bool value = true); @@ -74,16 +77,19 @@ class CoincenterCommand { bool isPercentageAmount() const { return _isPercentageAmount; } bool withBalanceInUse() const { return _withBalanceInUse; } + const ReplayOptions& replayOptions() const { return std::get(_specialOptions); } + bool operator==(const CoincenterCommand&) const noexcept = default; using trivially_relocatable = std::bool_constant && is_trivially_relocatable_v && is_trivially_relocatable_v && - is_trivially_relocatable_v && is_trivially_relocatable_v>::type; + is_trivially_relocatable_v && is_trivially_relocatable_v && + is_trivially_relocatable_v>::type; private: - using SpecialOptions = - std::variant; + using SpecialOptions = std::variant; ExchangeNames _exchangeNames; SpecialOptions _specialOptions; diff --git a/src/engine/include/coincenteroptions.hpp b/src/engine/include/coincenteroptions.hpp index eaa66bfb..24b44af3 100644 --- a/src/engine/include/coincenteroptions.hpp +++ b/src/engine/include/coincenteroptions.hpp @@ -9,7 +9,7 @@ #include "coincentercommandtype.hpp" #include "coincenteroptionsdef.hpp" #include "commandlineoption.hpp" -#include "exchangepublicapi.hpp" +#include "replay-options.hpp" #include "timedef.hpp" #include "tradedefinitions.hpp" #include "tradeoptions.hpp" @@ -30,6 +30,8 @@ class CoincenterCmdLineOptions { TradeOptions computeTradeOptions() const; WithdrawOptions computeWithdrawOptions() const; + ReplayOptions computeReplayOptions(Duration dur) const; + std::string_view getDataDir() const { return dataDir.empty() ? SelectDefaultDataDir() : dataDir; } std::pair getTradeArgStr() const; @@ -100,6 +102,13 @@ class CoincenterCmdLineOptions { std::string_view lastTrades; + std::string_view marketData; + + std::optional replay; + std::string_view algorithmNames; + std::string_view market; + std::optional replayMarkets; + CommandLineOptionalInt32 repeats; int32_t monitoringPort = CoincenterCmdLineOptionsDefinitions::kDefaultMonitoringPort; int32_t depth = kUndefinedDepth; @@ -114,6 +123,8 @@ class CoincenterCmdLineOptions { bool version = false; bool useMonitoring = false; bool withBalanceInUse = false; + bool validate = false; + bool validateOnly = false; bool operator==(const CoincenterCmdLineOptions&) const noexcept = default; diff --git a/src/engine/include/coincenteroptionsdef.hpp b/src/engine/include/coincenteroptionsdef.hpp index a27f1efc..aeb4c8e1 100644 --- a/src/engine/include/coincenteroptionsdef.hpp +++ b/src/engine/include/coincenteroptionsdef.hpp @@ -432,6 +432,50 @@ struct CoincenterAllowedOptions : private CoincenterCmdLineOptionsDefinitions { "Prints withdraw fees for matching currency and exchanges.\n" "Currency and exchanges are optional, if specified, output will be filtered to match them."}, &OptValueType::withdrawFees}, + {{{"Automation", 8000}, + "market-data", + "", + "Query last trades and order books of given market without printing the result on screen, for given exchanges " + "if specified.\n" + "This is the equivalent of calling last-trades and order-book but is useful combined with the repeat " + "command to store market data on disk."}, + &OptValueType::marketData}, + {{{"Automation", 8000}, + "replay-markets", + "", + "Print markets available for replay, that is, markets that have some data within the time " + "window {now - duration, now}."}, + &OptValueType::replayMarkets}, + {{{"Automation", 8001}, + "replay", + "", + "Replay algorithms on serialized, historical data." + "\nAll known algorithms will be replayed one by one, on all stored markets that have some data within the time " + "window {now - duration, now}. Use below flags to filter more precisely on which data to replay from."}, + &OptValueType::replay}, + {{{"Automation", 8002}, + "--algorithms", + "", + "Pick specific algorithm names to replay with. Default will replay with all known ones."}, + &OptValueType::algorithmNames}, + {{{"Automation", 8003}, + "--market", + "", + "Only replay for specific market. Default will replay all stored markets."}, + &OptValueType::market}, + {{{"Automation", 8003}, + "--validate", + "", + "Filter invalid data during replay.\nThis is disabled by default, use this option when you suspect that " + "invalid data may be present in the replayed time window."}, + &OptValueType::validate}, + {{{"Automation", 8003}, + "--validate-only", + "", + "Instead of launching replay algorithm, only validates serialized data." + "\nNominal replay will not validate input data to optimize performance, use this option to validate data once " + "and for all."}, + &OptValueType::validateOnly}, {{{"Monitoring", 9000}, "--monitoring", "", diff --git a/src/engine/include/exchangesorchestrator.hpp b/src/engine/include/exchangesorchestrator.hpp index a2c052dd..4498a709 100644 --- a/src/engine/include/exchangesorchestrator.hpp +++ b/src/engine/include/exchangesorchestrator.hpp @@ -6,14 +6,18 @@ #include "exchange-names.hpp" #include "exchangename.hpp" #include "exchangeretriever.hpp" +#include "market-trader-engine.hpp" #include "market.hpp" #include "queryresulttypes.hpp" #include "threadpool.hpp" +#include "time-window.hpp" #include "withdrawoptions.hpp" namespace cct { +class ReplayOptions; class RequestsConfig; + class ExchangesOrchestrator { public: using UniquePublicSelectedExchanges = ExchangeRetriever::UniquePublicSelectedExchanges; @@ -89,6 +93,19 @@ class ExchangesOrchestrator { MonetaryAmountPerExchange getLastPricePerExchange(Market mk, ExchangeNameSpan exchangeNames); + MarketDataPerExchange getMarketDataPerExchange(std::span marketPerPublicExchange, + ExchangeNameSpan exchangeNames); + + MarketTimestampSetsPerExchange pullAvailableMarketsForReplay(TimeWindow timeWindow, ExchangeNameSpan exchangeNames); + + MarketTradeRangeStatsPerExchange traderConsumeRange(const ReplayOptions &replayOptions, TimeWindow subTimeWindow, + std::span marketTraderEngines, + ExchangeNameSpan exchangeNames); + + MarketTradingGlobalResultPerExchange getMarketTraderResultPerExchange( + std::span marketTraderEngines, MarketTradeRangeStatsPerExchange &&tradeRangeStatsPerExchange, + ExchangeNameSpan exchangeNames); + private: ExchangeRetriever _exchangeRetriever; ThreadPool _threadPool; diff --git a/src/engine/include/query-result-type-helpers.hpp b/src/engine/include/query-result-type-helpers.hpp new file mode 100644 index 00000000..6db39046 --- /dev/null +++ b/src/engine/include/query-result-type-helpers.hpp @@ -0,0 +1,16 @@ +#pragma once + +#include "exchangepublicapitypes.hpp" +#include "market-timestamp-set.hpp" +#include "market.hpp" +#include "queryresulttypes.hpp" + +namespace cct { + +bool ContainsMarket(Market market, const MarketTimestampSet &marketTimestampSet); + +bool ContainsMarket(Market market, const MarketTimestampSets &marketTimestampSets); + +MarketSet ComputeAllMarkets(const MarketTimestampSetsPerExchange &marketTimestampSetsPerExchange); + +} // namespace cct \ No newline at end of file diff --git a/src/engine/include/queryresultprinter.hpp b/src/engine/include/queryresultprinter.hpp index 6c0a1f52..81e692ac 100644 --- a/src/engine/include/queryresultprinter.hpp +++ b/src/engine/include/queryresultprinter.hpp @@ -15,6 +15,7 @@ #include "ordersconstraints.hpp" #include "queryresulttypes.hpp" #include "simpletable.hpp" +#include "time-window.hpp" #include "withdrawsconstraints.hpp" namespace cct { @@ -66,7 +67,7 @@ class QueryResultPrinter { } void printClosedOrders(const ClosedOrdersPerExchange &closedOrdersPerExchange, - const OrdersConstraints &ordersConstraints) const; + const OrdersConstraints &ordersConstraints = OrdersConstraints{}) const; void printOpenedOrders(const OpenedOrdersPerExchange &openedOrdersPerExchange, const OrdersConstraints &ordersConstraints) const; @@ -104,6 +105,13 @@ class QueryResultPrinter { const TradedAmountsVectorWithFinalAmountPerExchange &tradedAmountsVectorWithFinalAmountPerExchange, CurrencyCode currencyCode) const; + void printMarketsForReplay(TimeWindow timeWindow, + const MarketTimestampSetsPerExchange &marketTimestampSetsPerExchange); + + void printMarketTradingResults(TimeWindow timeWindow, + const MarketTradingGlobalResultPerExchange &marketTradingResultPerExchange, + CoincenterCommandType commandType) const; + private: void printTrades(const TradeResultPerExchange &tradeResultPerExchange, MonetaryAmount amount, bool isPercentageTrade, CurrencyCode toCurrency, const TradeOptions &tradeOptions, CoincenterCommandType commandType) const; diff --git a/src/engine/include/queryresulttypes.hpp b/src/engine/include/queryresulttypes.hpp index 60f1028b..5af47573 100644 --- a/src/engine/include/queryresulttypes.hpp +++ b/src/engine/include/queryresulttypes.hpp @@ -13,10 +13,14 @@ #include "currencyexchangeflatset.hpp" #include "exchangeprivateapitypes.hpp" #include "exchangepublicapitypes.hpp" +#include "market-timestamp-set.hpp" +#include "market-trading-global-result.hpp" +#include "market-trading-result.hpp" #include "marketorderbook.hpp" #include "monetaryamount.hpp" #include "monetaryamountbycurrencyset.hpp" #include "public-trade-vector.hpp" +#include "trade-range-stats.hpp" #include "traderesult.hpp" #include "wallet.hpp" #include "withdrawinfo.hpp" @@ -31,6 +35,8 @@ using MarketOrderBookConversionRate = std::tuple; +using MarketPerExchange = FixedCapacityVector, kNbSupportedExchanges>; + using MarketsPerExchange = FixedCapacityVector, kNbSupportedExchanges>; using MonetaryAmountPerExchange = FixedCapacityVector, kNbSupportedExchanges>; @@ -40,6 +46,9 @@ using MonetaryAmountByCurrencySetPerExchange = using TradesPerExchange = FixedCapacityVector, kNbSupportedExchanges>; +using MarketDataPerExchange = + FixedCapacityVector>, kNbSupportedExchanges>; + using TradeResultPerExchange = SmallVector, kTypicalNbPrivateAccounts>; using TradedAmountsVectorWithFinalAmountPerExchange = @@ -68,4 +77,14 @@ using DeliveredWithdrawInfoWithExchanges = std::pair, kTypicalNbPrivateAccounts>; using ConversionPathPerExchange = FixedCapacityVector, kNbSupportedExchanges>; + +using MarketTimestampSetsPerExchange = FixedCapacityVector, kNbSupportedExchanges>; + +using MarketTradeRangeStatsPerExchange = FixedCapacityVector, kNbSupportedExchanges>; + +using MarketTradingResultPerExchange = FixedCapacityVector, kNbSupportedExchanges>; + +using MarketTradingGlobalResultPerExchange = + FixedCapacityVector, kNbSupportedExchanges>; + } // namespace cct diff --git a/src/engine/include/replay-algorithm-name-iterator.hpp b/src/engine/include/replay-algorithm-name-iterator.hpp new file mode 100644 index 00000000..21995286 --- /dev/null +++ b/src/engine/include/replay-algorithm-name-iterator.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include +#include +#include + +namespace cct { + +/// Convenient class to iterate on the algorithm names, comma separated. +/// If 'algorithmNames' is empty, it will loop on all available ones (given by 'allAlgorithms') +class ReplayAlgorithmNameIterator { + public: + ReplayAlgorithmNameIterator(std::string_view algorithmNames, std::span allAlgorithms); + + /// Returns true if and only if there is at least one additional algorithm name to iterate on. + bool hasNext() const; + + /// Get next algorithm name and advance the iterator. + /// Undefined behavior if 'hasNext' is 'false'. + std::string_view next(); + + private: + std::span _allAlgorithms; + std::string_view _algorithmNames; + int32_t _begPos; + int32_t _endPos; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/engine/include/replay-options.hpp b/src/engine/include/replay-options.hpp new file mode 100644 index 00000000..44a762a2 --- /dev/null +++ b/src/engine/include/replay-options.hpp @@ -0,0 +1,33 @@ +#pragma once + +#include +#include + +#include "time-window.hpp" + +namespace cct { + +class ReplayOptions { + public: + enum class ReplayMode : int8_t { kValidateOnly, kCheckedLaunchAlgorithm, kUncheckedLaunchAlgorithm }; + + ReplayOptions() noexcept = default; + + /// Algorithm names should be comma separated. Empty string will match all. + ReplayOptions(TimeWindow timeWindow, std::string_view algorithmNames, ReplayMode replayMode); + + TimeWindow timeWindow() const { return _timeWindow; } + + std::string_view algorithmNames() const; + + ReplayMode replayMode() const { return _replayMode; } + + bool operator==(const ReplayOptions &) const noexcept = default; + + private: + TimeWindow _timeWindow; + std::string_view _algorithmNames; + ReplayMode _replayMode; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/engine/include/stringoptionparser.hpp b/src/engine/include/stringoptionparser.hpp index 58bed922..330d3aeb 100644 --- a/src/engine/include/stringoptionparser.hpp +++ b/src/engine/include/stringoptionparser.hpp @@ -10,6 +10,7 @@ #include "exchange-names.hpp" #include "market.hpp" #include "monetaryamount.hpp" +#include "timedef.hpp" namespace cct { class StringOptionParser { @@ -29,6 +30,10 @@ class StringOptionParser { /// after the currency CurrencyCode parseCurrency(FieldIs fieldIs = FieldIs::kMandatory, char delimiter = ','); + /// If FieldIs is kOptional and there is no duration, kUndefinedDuration duration will be returned. + /// otherwise exception invalid_argument will be raised + Duration parseDuration(FieldIs fieldIs = FieldIs::kMandatory); + /// If FieldIs is kOptional and there is no market, default market will be returned. /// otherwise exception invalid_argument will be raised. /// @param delimiter defines the expected character (could be not present, which means end of parsing) @@ -55,4 +60,4 @@ class StringOptionParser { std::string_view _opt; std::string_view::size_type _pos{}; }; -} // namespace cct \ No newline at end of file +} // namespace cct diff --git a/src/engine/src/coincenter-commands-iterator.cpp b/src/engine/src/coincenter-commands-iterator.cpp new file mode 100644 index 00000000..f0f773bd --- /dev/null +++ b/src/engine/src/coincenter-commands-iterator.cpp @@ -0,0 +1,74 @@ +#include "coincenter-commands-iterator.hpp" + +#include + +#include "cct_const.hpp" +#include "coincentercommandtype.hpp" +#include "exchangename.hpp" + +namespace cct { + +CoincenterCommandsIterator::CoincenterCommandsIterator(CoincenterCommandSpan commands) noexcept + : _commands(commands), _pos() {} + +namespace { +using PublicExchangePresenceBitset = std::bitset; + +bool UpdateBitsetAreNewExchanges(const CoincenterCommand &command, + PublicExchangePresenceBitset &publicExchangePresence) { + if (command.exchangeNames().empty()) { + // All public exchanges used + const auto result = publicExchangePresence.none(); + publicExchangePresence.set(); + return result; + } + for (const ExchangeName &exchangeName : command.exchangeNames()) { + const auto exchangePos = exchangeName.publicExchangePos(); + if (publicExchangePresence[exchangePos]) { + return false; + } + publicExchangePresence.set(exchangePos); + } + return true; +} + +bool CommandTypeCanBeGrouped(CoincenterCommandType type) { + // Compatible command types need to be explicitly set + // For now, only market data is compatible + switch (type) { + case CoincenterCommandType::kMarketData: + return true; + default: + return false; + } +} + +} // namespace + +bool CoincenterCommandsIterator::hasNextCommandGroup() const { return _pos < _commands.size(); } + +CoincenterCommandsIterator::CoincenterCommandSpan CoincenterCommandsIterator::nextCommandGroup() { + CoincenterCommandSpan groupedCommands(_commands.begin() + _pos, 1U); + + if (CommandTypeCanBeGrouped(groupedCommands.front().type())) { + PublicExchangePresenceBitset publicExchangePresence; + UpdateBitsetAreNewExchanges(groupedCommands.front(), publicExchangePresence); + + while (_pos + groupedCommands.size() < _commands.size()) { + const CoincenterCommand &nextCommand = _commands[_pos + groupedCommands.size()]; + if (nextCommand.type() != groupedCommands.front().type()) { + break; + } + if (!UpdateBitsetAreNewExchanges(nextCommand, publicExchangePresence)) { + break; + } + // Add new command to group + groupedCommands = CoincenterCommandSpan(groupedCommands.data(), groupedCommands.size() + 1); + } + } + + _pos += groupedCommands.size(); + return groupedCommands; +} + +} // namespace cct \ No newline at end of file diff --git a/src/engine/src/coincenter.cpp b/src/engine/src/coincenter.cpp index 1a08e76e..8dbbfa08 100644 --- a/src/engine/src/coincenter.cpp +++ b/src/engine/src/coincenter.cpp @@ -13,6 +13,7 @@ #include "cct_exception.hpp" #include "cct_invalid_argument_exception.hpp" #include "cct_log.hpp" +#include "coincenter-commands-iterator.hpp" #include "coincentercommand.hpp" #include "coincentercommands.hpp" #include "coincentercommandtype.hpp" @@ -25,11 +26,18 @@ #include "exchangepublicapi.hpp" #include "exchangeretriever.hpp" #include "exchangesecretsinfo.hpp" +#include "market-timestamp-set.hpp" +#include "market-trader-engine.hpp" +#include "market-trader-factory.hpp" #include "market.hpp" #include "monetaryamount.hpp" #include "ordersconstraints.hpp" +#include "query-result-type-helpers.hpp" #include "queryresultprinter.hpp" #include "queryresulttypes.hpp" +#include "replay-algorithm-name-iterator.hpp" +#include "replay-options.hpp" +#include "time-window.hpp" #include "timedef.hpp" #include "transferablecommandresult.hpp" #include "withdrawsconstraints.hpp" @@ -113,35 +121,40 @@ int Coincenter::process(const CoincenterCommands &coincenterCommands) { } } TransferableCommandResultVector transferableResults; - for (const auto &cmd : commands) { - transferableResults = processCommand(cmd, transferableResults); + CoincenterCommandsIterator commandsIterator(commands); + while (commandsIterator.hasNextCommandGroup()) { + const auto groupedCommands = commandsIterator.nextCommandGroup(); + transferableResults = processGroupedCommands(groupedCommands, transferableResults); ++nbCommandsProcessed; } } return nbCommandsProcessed; } -TransferableCommandResultVector Coincenter::processCommand( - const CoincenterCommand &cmd, std::span previousTransferableResults) { +TransferableCommandResultVector Coincenter::processGroupedCommands( + std::span groupedCommands, + std::span previousTransferableResults) { TransferableCommandResultVector transferableResults; - switch (cmd.type()) { + const auto &firstCmd = groupedCommands.front(); + // All grouped commands have same type - logic to handle multiple commands in a group should be handled per use case + switch (firstCmd.type()) { case CoincenterCommandType::kHealthCheck: { - const auto healthCheckStatus = healthCheck(cmd.exchangeNames()); + const auto healthCheckStatus = healthCheck(firstCmd.exchangeNames()); _queryResultPrinter.printHealthCheck(healthCheckStatus); break; } case CoincenterCommandType::kCurrencies: { - const auto currenciesPerExchange = getCurrenciesPerExchange(cmd.exchangeNames()); + const auto currenciesPerExchange = getCurrenciesPerExchange(firstCmd.exchangeNames()); _queryResultPrinter.printCurrencies(currenciesPerExchange); break; } case CoincenterCommandType::kMarkets: { - const auto marketsPerExchange = getMarketsPerExchange(cmd.cur1(), cmd.cur2(), cmd.exchangeNames()); - _queryResultPrinter.printMarkets(cmd.cur1(), cmd.cur2(), marketsPerExchange, cmd.type()); + const auto marketsPerExchange = getMarketsPerExchange(firstCmd.cur1(), firstCmd.cur2(), firstCmd.exchangeNames()); + _queryResultPrinter.printMarkets(firstCmd.cur1(), firstCmd.cur2(), marketsPerExchange, firstCmd.type()); break; } case CoincenterCommandType::kConversion: { - if (cmd.amount().isDefault()) { + if (firstCmd.amount().isDefault()) { std::array startAmountsPerExchangePos; bool oneSet = false; for (const auto &transferableResult : previousTransferableResults) { @@ -158,90 +171,96 @@ TransferableCommandResultVector Coincenter::processCommand( throw invalid_argument("Missing input amount to convert from"); } - const auto conversionPerExchange = getConversion(startAmountsPerExchangePos, cmd.cur1(), cmd.exchangeNames()); - _queryResultPrinter.printConversion(startAmountsPerExchangePos, cmd.cur1(), conversionPerExchange); + const auto conversionPerExchange = + getConversion(startAmountsPerExchangePos, firstCmd.cur1(), firstCmd.exchangeNames()); + _queryResultPrinter.printConversion(startAmountsPerExchangePos, firstCmd.cur1(), conversionPerExchange); FillConversionTransferableCommandResults(conversionPerExchange, transferableResults); } else { - const auto conversionPerExchange = getConversion(cmd.amount(), cmd.cur1(), cmd.exchangeNames()); - _queryResultPrinter.printConversion(cmd.amount(), cmd.cur1(), conversionPerExchange); + const auto conversionPerExchange = getConversion(firstCmd.amount(), firstCmd.cur1(), firstCmd.exchangeNames()); + _queryResultPrinter.printConversion(firstCmd.amount(), firstCmd.cur1(), conversionPerExchange); FillConversionTransferableCommandResults(conversionPerExchange, transferableResults); } break; } case CoincenterCommandType::kConversionPath: { - const auto conversionPathPerExchange = getConversionPaths(cmd.market(), cmd.exchangeNames()); - _queryResultPrinter.printConversionPath(cmd.market(), conversionPathPerExchange); + const auto conversionPathPerExchange = getConversionPaths(firstCmd.market(), firstCmd.exchangeNames()); + _queryResultPrinter.printConversionPath(firstCmd.market(), conversionPathPerExchange); break; } case CoincenterCommandType::kLastPrice: { - const auto lastPricePerExchange = getLastPricePerExchange(cmd.market(), cmd.exchangeNames()); - _queryResultPrinter.printLastPrice(cmd.market(), lastPricePerExchange); + const auto lastPricePerExchange = getLastPricePerExchange(firstCmd.market(), firstCmd.exchangeNames()); + _queryResultPrinter.printLastPrice(firstCmd.market(), lastPricePerExchange); break; } case CoincenterCommandType::kTicker: { - const auto exchangeTickerMaps = getTickerInformation(cmd.exchangeNames()); + const auto exchangeTickerMaps = getTickerInformation(firstCmd.exchangeNames()); _queryResultPrinter.printTickerInformation(exchangeTickerMaps); break; } case CoincenterCommandType::kOrderbook: { const auto marketOrderBooksConversionRates = - getMarketOrderBooks(cmd.market(), cmd.exchangeNames(), cmd.cur1(), cmd.optDepth()); - _queryResultPrinter.printMarketOrderBooks(cmd.market(), cmd.cur1(), cmd.optDepth(), + getMarketOrderBooks(firstCmd.market(), firstCmd.exchangeNames(), firstCmd.cur1(), firstCmd.optDepth()); + _queryResultPrinter.printMarketOrderBooks(firstCmd.market(), firstCmd.cur1(), firstCmd.optDepth(), marketOrderBooksConversionRates); break; } case CoincenterCommandType::kLastTrades: { - const auto lastTradesPerExchange = getLastTradesPerExchange(cmd.market(), cmd.exchangeNames(), cmd.optDepth()); - _queryResultPrinter.printLastTrades(cmd.market(), cmd.optDepth(), lastTradesPerExchange); + const auto lastTradesPerExchange = + getLastTradesPerExchange(firstCmd.market(), firstCmd.exchangeNames(), firstCmd.optDepth()); + _queryResultPrinter.printLastTrades(firstCmd.market(), firstCmd.optDepth(), lastTradesPerExchange); break; } case CoincenterCommandType::kLast24hTradedVolume: { - const auto tradedVolumePerExchange = getLast24hTradedVolumePerExchange(cmd.market(), cmd.exchangeNames()); - _queryResultPrinter.printLast24hTradedVolume(cmd.market(), tradedVolumePerExchange); + const auto tradedVolumePerExchange = + getLast24hTradedVolumePerExchange(firstCmd.market(), firstCmd.exchangeNames()); + _queryResultPrinter.printLast24hTradedVolume(firstCmd.market(), tradedVolumePerExchange); break; } case CoincenterCommandType::kWithdrawFees: { - const auto withdrawFeesPerExchange = getWithdrawFees(cmd.cur1(), cmd.exchangeNames()); - _queryResultPrinter.printWithdrawFees(withdrawFeesPerExchange, cmd.cur1()); + const auto withdrawFeesPerExchange = getWithdrawFees(firstCmd.cur1(), firstCmd.exchangeNames()); + _queryResultPrinter.printWithdrawFees(withdrawFeesPerExchange, firstCmd.cur1()); break; } case CoincenterCommandType::kBalance: { - const auto amountIncludePolicy = cmd.withBalanceInUse() ? BalanceOptions::AmountIncludePolicy::kWithBalanceInUse - : BalanceOptions::AmountIncludePolicy::kOnlyAvailable; - const BalanceOptions balanceOptions(amountIncludePolicy, cmd.cur1()); - const auto balancePerExchange = getBalance(cmd.exchangeNames(), balanceOptions); - _queryResultPrinter.printBalance(balancePerExchange, cmd.cur1()); + const auto amountIncludePolicy = firstCmd.withBalanceInUse() + ? BalanceOptions::AmountIncludePolicy::kWithBalanceInUse + : BalanceOptions::AmountIncludePolicy::kOnlyAvailable; + const BalanceOptions balanceOptions(amountIncludePolicy, firstCmd.cur1()); + const auto balancePerExchange = getBalance(firstCmd.exchangeNames(), balanceOptions); + _queryResultPrinter.printBalance(balancePerExchange, firstCmd.cur1()); break; } case CoincenterCommandType::kDepositInfo: { - const auto walletPerExchange = getDepositInfo(cmd.exchangeNames(), cmd.cur1()); - _queryResultPrinter.printDepositInfo(cmd.cur1(), walletPerExchange); + const auto walletPerExchange = getDepositInfo(firstCmd.exchangeNames(), firstCmd.cur1()); + _queryResultPrinter.printDepositInfo(firstCmd.cur1(), walletPerExchange); break; } case CoincenterCommandType::kOrdersClosed: { - const auto closedOrdersPerExchange = getClosedOrders(cmd.exchangeNames(), cmd.ordersConstraints()); - _queryResultPrinter.printClosedOrders(closedOrdersPerExchange, cmd.ordersConstraints()); + const auto closedOrdersPerExchange = getClosedOrders(firstCmd.exchangeNames(), firstCmd.ordersConstraints()); + _queryResultPrinter.printClosedOrders(closedOrdersPerExchange, firstCmd.ordersConstraints()); break; } case CoincenterCommandType::kOrdersOpened: { - const auto openedOrdersPerExchange = getOpenedOrders(cmd.exchangeNames(), cmd.ordersConstraints()); - _queryResultPrinter.printOpenedOrders(openedOrdersPerExchange, cmd.ordersConstraints()); + const auto openedOrdersPerExchange = getOpenedOrders(firstCmd.exchangeNames(), firstCmd.ordersConstraints()); + _queryResultPrinter.printOpenedOrders(openedOrdersPerExchange, firstCmd.ordersConstraints()); break; } case CoincenterCommandType::kOrdersCancel: { - const auto nbCancelledOrdersPerExchange = cancelOrders(cmd.exchangeNames(), cmd.ordersConstraints()); - _queryResultPrinter.printCancelledOrders(nbCancelledOrdersPerExchange, cmd.ordersConstraints()); + const auto nbCancelledOrdersPerExchange = cancelOrders(firstCmd.exchangeNames(), firstCmd.ordersConstraints()); + _queryResultPrinter.printCancelledOrders(nbCancelledOrdersPerExchange, firstCmd.ordersConstraints()); break; } case CoincenterCommandType::kRecentDeposits: { - const auto depositsPerExchange = getRecentDeposits(cmd.exchangeNames(), cmd.withdrawsOrDepositsConstraints()); - _queryResultPrinter.printRecentDeposits(depositsPerExchange, cmd.withdrawsOrDepositsConstraints()); + const auto depositsPerExchange = + getRecentDeposits(firstCmd.exchangeNames(), firstCmd.withdrawsOrDepositsConstraints()); + _queryResultPrinter.printRecentDeposits(depositsPerExchange, firstCmd.withdrawsOrDepositsConstraints()); break; } case CoincenterCommandType::kRecentWithdraws: { - const auto withdrawsPerExchange = getRecentWithdraws(cmd.exchangeNames(), cmd.withdrawsOrDepositsConstraints()); - _queryResultPrinter.printRecentWithdraws(withdrawsPerExchange, cmd.withdrawsOrDepositsConstraints()); + const auto withdrawsPerExchange = + getRecentWithdraws(firstCmd.exchangeNames(), firstCmd.withdrawsOrDepositsConstraints()); + _queryResultPrinter.printRecentWithdraws(withdrawsPerExchange, firstCmd.withdrawsOrDepositsConstraints()); break; } case CoincenterCommandType::kTrade: { @@ -249,50 +268,83 @@ TransferableCommandResultVector Coincenter::processCommand( // - standard full information with an amount to trade, a destination currency and an optional list of exchanges // where to trade // - a currency - the destination one, and start amount and exchange(s) should come from previous command result - auto [startAmount, exchangeNames] = ComputeTradeAmountAndExchanges(cmd, previousTransferableResults); + auto [startAmount, exchangeNames] = ComputeTradeAmountAndExchanges(firstCmd, previousTransferableResults); if (startAmount.isDefault()) { break; } const auto tradeResultPerExchange = - trade(startAmount, cmd.isPercentageAmount(), cmd.cur1(), exchangeNames, cmd.tradeOptions()); - _queryResultPrinter.printTrades(tradeResultPerExchange, startAmount, cmd.isPercentageAmount(), cmd.cur1(), - cmd.tradeOptions()); + trade(startAmount, firstCmd.isPercentageAmount(), firstCmd.cur1(), exchangeNames, firstCmd.tradeOptions()); + _queryResultPrinter.printTrades(tradeResultPerExchange, startAmount, firstCmd.isPercentageAmount(), + firstCmd.cur1(), firstCmd.tradeOptions()); FillTradeTransferableCommandResults(tradeResultPerExchange, transferableResults); break; } case CoincenterCommandType::kBuy: { - const auto tradeResultPerExchange = smartBuy(cmd.amount(), cmd.exchangeNames(), cmd.tradeOptions()); - _queryResultPrinter.printBuyTrades(tradeResultPerExchange, cmd.amount(), cmd.tradeOptions()); + const auto tradeResultPerExchange = + smartBuy(firstCmd.amount(), firstCmd.exchangeNames(), firstCmd.tradeOptions()); + _queryResultPrinter.printBuyTrades(tradeResultPerExchange, firstCmd.amount(), firstCmd.tradeOptions()); FillTradeTransferableCommandResults(tradeResultPerExchange, transferableResults); break; } case CoincenterCommandType::kSell: { - auto [startAmount, exchangeNames] = ComputeTradeAmountAndExchanges(cmd, previousTransferableResults); + auto [startAmount, exchangeNames] = ComputeTradeAmountAndExchanges(firstCmd, previousTransferableResults); if (startAmount.isDefault()) { break; } const auto tradeResultPerExchange = - smartSell(startAmount, cmd.isPercentageAmount(), exchangeNames, cmd.tradeOptions()); - _queryResultPrinter.printSellTrades(tradeResultPerExchange, cmd.amount(), cmd.isPercentageAmount(), - cmd.tradeOptions()); + smartSell(startAmount, firstCmd.isPercentageAmount(), exchangeNames, firstCmd.tradeOptions()); + _queryResultPrinter.printSellTrades(tradeResultPerExchange, firstCmd.amount(), firstCmd.isPercentageAmount(), + firstCmd.tradeOptions()); FillTradeTransferableCommandResults(tradeResultPerExchange, transferableResults); break; } case CoincenterCommandType::kWithdrawApply: { - const auto [grossAmount, exchangeName] = ComputeWithdrawAmount(cmd, previousTransferableResults); + const auto [grossAmount, exchangeName] = ComputeWithdrawAmount(firstCmd, previousTransferableResults); if (grossAmount.isDefault()) { break; } - const auto deliveredWithdrawInfoWithExchanges = withdraw(grossAmount, cmd.isPercentageAmount(), exchangeName, - cmd.exchangeNames().back(), cmd.withdrawOptions()); - _queryResultPrinter.printWithdraw(deliveredWithdrawInfoWithExchanges, cmd.isPercentageAmount(), - cmd.withdrawOptions()); + const auto deliveredWithdrawInfoWithExchanges = + withdraw(grossAmount, firstCmd.isPercentageAmount(), exchangeName, firstCmd.exchangeNames().back(), + firstCmd.withdrawOptions()); + _queryResultPrinter.printWithdraw(deliveredWithdrawInfoWithExchanges, firstCmd.isPercentageAmount(), + firstCmd.withdrawOptions()); transferableResults.emplace_back(deliveredWithdrawInfoWithExchanges.first[1]->createExchangeName(), deliveredWithdrawInfoWithExchanges.second.receivedAmount()); break; } case CoincenterCommandType::kDustSweeper: { - _queryResultPrinter.printDustSweeper(dustSweeper(cmd.exchangeNames(), cmd.cur1()), cmd.cur1()); + _queryResultPrinter.printDustSweeper(dustSweeper(firstCmd.exchangeNames(), firstCmd.cur1()), firstCmd.cur1()); + break; + } + case CoincenterCommandType::kMarketData: { + std::array marketPerPublicExchange; + for (const auto &cmd : groupedCommands) { + if (cmd.exchangeNames().empty()) { + std::ranges::fill(marketPerPublicExchange, cmd.market()); + } else { + for (const auto &exchangeName : cmd.exchangeNames()) { + marketPerPublicExchange[exchangeName.publicExchangePos()] = cmd.market(); + } + } + } + // No return value here, this command is made only for storing purposes. + queryMarketDataPerExchange(marketPerPublicExchange); + break; + } + case CoincenterCommandType::kReplay: { + /// This implementation of AbstractMarketTraderFactory is only provided as an example. + /// You can extend coincenter library and: + /// - Provide your own algorithms by implementing your own MarketTraderFactory will all your algorithms. + /// - Create your own CommandType that will call coincenter.replay with the same parameters as below, with your + /// own MarketTraderFactory. + MarketTraderFactory marketTraderFactory; + replay(marketTraderFactory, firstCmd.replayOptions(), firstCmd.market(), firstCmd.exchangeNames()); + break; + } + case CoincenterCommandType::kReplayMarkets: { + const auto marketTimestampSetsPerExchange = + getMarketsAvailableForReplay(firstCmd.replayOptions(), firstCmd.exchangeNames()); + _queryResultPrinter.printMarketsForReplay(firstCmd.replayOptions().timeWindow(), marketTimestampSetsPerExchange); break; } default: @@ -327,6 +379,40 @@ MarketOrderBookConversionRates Coincenter::getMarketOrderBooks(Market mk, Exchan return ret; } +void Coincenter::queryMarketDataPerExchange(std::span marketPerPublicExchange) { + ExchangeNames exchangeNames; + + int exchangePos{}; + for (Market market : marketPerPublicExchange) { + if (market.isDefined()) { + exchangeNames.emplace_back(kSupportedExchanges[exchangePos]); + } + ++exchangePos; + } + + const auto marketDataPerExchange = + _exchangesOrchestrator.getMarketDataPerExchange(marketPerPublicExchange, exchangeNames); + + // Transform data structures to export metrics input format + MarketOrderBookConversionRates marketOrderBookConversionRates(marketDataPerExchange.size()); + TradesPerExchange lastTradesPerExchange(marketDataPerExchange.size()); + + std::ranges::transform(marketDataPerExchange, marketOrderBookConversionRates.begin(), + [](const auto &exchangeWithPairOrderBooksAndTrades) { + return std::make_tuple(exchangeWithPairOrderBooksAndTrades.first->name(), + exchangeWithPairOrderBooksAndTrades.second.first, std::nullopt); + }); + + std::ranges::transform(marketDataPerExchange, lastTradesPerExchange.begin(), + [](const auto &exchangeWithPairOrderBooksAndTrades) { + return std::make_pair(exchangeWithPairOrderBooksAndTrades.first, + exchangeWithPairOrderBooksAndTrades.second.second); + }); + + _metricsExporter.exportOrderbookMetrics(marketOrderBookConversionRates); + _metricsExporter.exportLastTradesMetrics(lastTradesPerExchange); +} + BalancePerExchange Coincenter::getBalance(std::span privateExchangeNames, const BalanceOptions &balanceOptions) { CurrencyCode equiCurrency = balanceOptions.equiCurrency(); @@ -459,6 +545,202 @@ MonetaryAmountPerExchange Coincenter::getLastPricePerExchange(Market mk, Exchang return _exchangesOrchestrator.getLastPricePerExchange(mk, exchangeNames); } +MarketTimestampSetsPerExchange Coincenter::getMarketsAvailableForReplay(const ReplayOptions &replayOptions, + ExchangeNameSpan exchangeNames) { + return _exchangesOrchestrator.pullAvailableMarketsForReplay(replayOptions.timeWindow(), exchangeNames); +} + +namespace { +auto CreateExchangeNameVector(Market market, const MarketTimestampSetsPerExchange &marketTimestampSetsPerExchange) { + PublicExchangeNameVector exchangesWithThisMarketData; + for (const auto &[exchange, marketTimestampSets] : marketTimestampSetsPerExchange) { + if (ContainsMarket(market, marketTimestampSets)) { + exchangesWithThisMarketData.emplace_back(exchange->name()); + } + } + return exchangesWithThisMarketData; +} + +void CreateAndRegisterTraderAlgorithms(const AbstractMarketTraderFactory &marketTraderFactory, + std::string_view algorithmName, + std::span marketTraderEngines) { + for (auto &marketTraderEngine : marketTraderEngines) { + const auto &marketTraderEngineState = marketTraderEngine.marketTraderEngineState(); + + marketTraderEngine.registerMarketTrader(marketTraderFactory.construct(algorithmName, marketTraderEngineState)); + } +} + +bool Filter(Market market, MarketTimestampSet &marketTimestampSet) { + auto it = std::partition_point(marketTimestampSet.begin(), marketTimestampSet.end(), + [market](const auto &marketTimestamp) { return marketTimestamp.market < market; }); + if (it != marketTimestampSet.end() && it->market == market) { + auto marketTimestamp = *it; + marketTimestampSet.clear(); + marketTimestampSet.insert(marketTimestamp); + return false; + } + + marketTimestampSet.clear(); + return true; +} + +void Filter(Market market, MarketTimestampSetsPerExchange &marketTimestampSetsPerExchange) { + for (auto it = marketTimestampSetsPerExchange.begin(); it != marketTimestampSetsPerExchange.end();) { + const bool orderBooksEmpty = Filter(market, it->second.orderBooksMarkets); + const bool tradesEmpty = Filter(market, it->second.tradesMarkets); + + if (orderBooksEmpty && tradesEmpty) { + // no more data, remove the exchange entry completely + it = marketTimestampSetsPerExchange.erase(it); + } else { + ++it; + } + } +} + +} // namespace + +void Coincenter::replay(const AbstractMarketTraderFactory &marketTraderFactory, const ReplayOptions &replayOptions, + Market market, ExchangeNameSpan exchangeNames) { + const TimeWindow timeWindow = replayOptions.timeWindow(); + auto marketTimestampSetsPerExchange = _exchangesOrchestrator.pullAvailableMarketsForReplay(timeWindow, exchangeNames); + + if (market.isDefined()) { + Filter(market, marketTimestampSetsPerExchange); + } + + MarketSet allMarkets = ComputeAllMarkets(marketTimestampSetsPerExchange); + + ReplayAlgorithmNameIterator replayAlgorithmNameIterator(replayOptions.algorithmNames(), + marketTraderFactory.allSupportedAlgorithms()); + + while (replayAlgorithmNameIterator.hasNext()) { + std::string_view algorithmName = replayAlgorithmNameIterator.next(); + + for (const Market replayMarket : allMarkets) { + auto exchangesWithThisMarketData = CreateExchangeNameVector(replayMarket, marketTimestampSetsPerExchange); + + // Create the MarketTraderEngines based on this market, filtering out exchanges without available amount to + // trade + MarketTraderEngineVector marketTraderEngines = + createMarketTraderEngines(replayOptions, replayMarket, exchangesWithThisMarketData); + + replayAlgorithm(marketTraderFactory, algorithmName, replayOptions, marketTraderEngines, + exchangesWithThisMarketData); + } + } +} + +void Coincenter::replayAlgorithm(const AbstractMarketTraderFactory &marketTraderFactory, std::string_view algorithmName, + const ReplayOptions &replayOptions, std::span marketTraderEngines, + const PublicExchangeNameVector &exchangesWithThisMarketData) { + CreateAndRegisterTraderAlgorithms(marketTraderFactory, algorithmName, marketTraderEngines); + + MarketTradeRangeStatsPerExchange tradeRangeStatsPerExchange = + tradingProcess(replayOptions, marketTraderEngines, exchangesWithThisMarketData); + + // Finally retrieve and print results for this market + MarketTradingGlobalResultPerExchange marketTradingResultPerExchange = + _exchangesOrchestrator.getMarketTraderResultPerExchange( + marketTraderEngines, std::move(tradeRangeStatsPerExchange), exchangesWithThisMarketData); + + _queryResultPrinter.printMarketTradingResults(replayOptions.timeWindow(), marketTradingResultPerExchange, + CoincenterCommandType::kReplay); +} + +namespace { +MonetaryAmount ComputeStartAmount(CurrencyCode currencyCode, MonetaryAmount convertedAmount) { + MonetaryAmount startAmount = convertedAmount; + + if (startAmount.currencyCode() != currencyCode) { + // This is possible as conversion may use equivalent fiats and stable coins + log::info("Target converted currency is different from market one, replace with market currency {} -> {}", + startAmount.currencyCode(), currencyCode); + startAmount = MonetaryAmount(startAmount.amount(), currencyCode, startAmount.nbDecimals()); + } + + return startAmount; +} +} // namespace + +Coincenter::MarketTraderEngineVector Coincenter::createMarketTraderEngines( + const ReplayOptions &replayOptions, Market market, PublicExchangeNameVector &exchangesWithThisMarketData) { + auto nbExchanges = exchangesWithThisMarketData.size(); + + const auto &automationConfig = _coincenterInfo.generalConfig().tradingConfig().automationConfig(); + const auto startBaseAmountEquivalent = automationConfig.startBaseAmountEquivalent(); + const auto startQuoteAmountEquivalent = automationConfig.startQuoteAmountEquivalent(); + const bool isValidateOnly = replayOptions.replayMode() == ReplayOptions::ReplayMode::kValidateOnly; + + auto convertedBaseAmountPerExchange = + isValidateOnly ? MonetaryAmountPerExchange{} + : getConversion(startBaseAmountEquivalent, market.base(), exchangesWithThisMarketData); + auto convertedQuoteAmountPerExchange = + isValidateOnly ? MonetaryAmountPerExchange{} + : getConversion(startQuoteAmountEquivalent, market.quote(), exchangesWithThisMarketData); + + MarketTraderEngineVector marketTraderEngines; + for (decltype(nbExchanges) exchangePos = 0; exchangePos < nbExchanges; ++exchangePos) { + const MonetaryAmount startBaseAmount = + isValidateOnly ? MonetaryAmount{0, market.base()} + : ComputeStartAmount(market.base(), convertedBaseAmountPerExchange[exchangePos].second); + const MonetaryAmount startQuoteAmount = + isValidateOnly ? MonetaryAmount{0, market.quote()} + : ComputeStartAmount(market.quote(), convertedQuoteAmountPerExchange[exchangePos].second); + + if (!isValidateOnly && (startBaseAmount == 0 || startQuoteAmount == 0)) { + log::warn("Cannot convert to start base / quote amounts for {} ({} / {})", + exchangesWithThisMarketData[exchangePos], startBaseAmount, startQuoteAmount); + exchangesWithThisMarketData.erase(exchangesWithThisMarketData.begin() + exchangePos); + convertedBaseAmountPerExchange.erase(convertedBaseAmountPerExchange.begin() + exchangePos); + convertedQuoteAmountPerExchange.erase(convertedQuoteAmountPerExchange.begin() + exchangePos); + --exchangePos; + --nbExchanges; + continue; + } + + const ExchangeConfig &exchangeConfig = + _coincenterInfo.exchangeConfig(exchangesWithThisMarketData[exchangePos].name()); + + marketTraderEngines.emplace_back(exchangeConfig, market, startBaseAmount, startQuoteAmount); + } + return marketTraderEngines; +} + +MarketTradeRangeStatsPerExchange Coincenter::tradingProcess(const ReplayOptions &replayOptions, + std::span marketTraderEngines, + ExchangeNameSpan exchangesWithThisMarketData) { + const auto &automationConfig = _coincenterInfo.generalConfig().tradingConfig().automationConfig(); + const auto loadChunkDuration = automationConfig.loadChunkDuration(); + const auto timeWindow = replayOptions.timeWindow(); + + MarketTradeRangeStatsPerExchange tradeRangeResultsPerExchange; + + // Main loop - parallelized by exchange, with time window chunks of loadChunkDuration + + TimeWindow subTimeWindow(timeWindow.from(), loadChunkDuration); + while (subTimeWindow.overlaps(timeWindow)) { + auto subRangeResultsPerExchange = _exchangesOrchestrator.traderConsumeRange( + replayOptions, subTimeWindow, marketTraderEngines, exchangesWithThisMarketData); + + if (tradeRangeResultsPerExchange.empty()) { + tradeRangeResultsPerExchange = std::move(subRangeResultsPerExchange); + } else { + int pos{}; + for (auto &[exchange, result] : subRangeResultsPerExchange) { + tradeRangeResultsPerExchange[pos].second += result; + ++pos; + } + } + + // Go to next sub time window + subTimeWindow = TimeWindow(subTimeWindow.to(), loadChunkDuration); + } + + return tradeRangeResultsPerExchange; +} + void Coincenter::updateFileCaches() const { log::debug("Store all cache files"); diff --git a/src/engine/src/coincentercommand.cpp b/src/engine/src/coincentercommand.cpp index a5bab733..b578dc93 100644 --- a/src/engine/src/coincentercommand.cpp +++ b/src/engine/src/coincentercommand.cpp @@ -11,6 +11,7 @@ #include "market.hpp" #include "monetaryamount.hpp" #include "ordersconstraints.hpp" +#include "timedef.hpp" #include "tradeoptions.hpp" #include "withdrawoptions.hpp" #include "withdrawsconstraints.hpp" @@ -124,4 +125,10 @@ CoincenterCommand& CoincenterCommand::withBalanceInUse(bool value) { _withBalanceInUse = value; return *this; } + +CoincenterCommand& CoincenterCommand::setReplayOptions(ReplayOptions replayOptions) { + _specialOptions = std::move(replayOptions); + return *this; +} + } // namespace cct diff --git a/src/engine/src/coincentercommands.cpp b/src/engine/src/coincentercommands.cpp index 3d4550aa..f819a993 100644 --- a/src/engine/src/coincentercommands.cpp +++ b/src/engine/src/coincentercommands.cpp @@ -11,7 +11,10 @@ #include "coincenteroptions.hpp" #include "currencycode.hpp" #include "depositsconstraints.hpp" +#include "market.hpp" +#include "replay-options.hpp" #include "stringoptionparser.hpp" +#include "time-window.hpp" #include "timedef.hpp" #include "withdrawsconstraints.hpp" @@ -199,6 +202,46 @@ void CoincenterCommands::addOption(const CoincenterCmdLineOptions &cmdLineOption .setExchangeNames(optionParser.parseExchanges()); } + if (!cmdLineOptions.marketData.empty()) { + optionParser = StringOptionParser(cmdLineOptions.marketData); + + _commands.emplace_back(CoincenterCommandType::kMarketData) + .setMarket(optionParser.parseMarket()) + .setExchangeNames(optionParser.parseExchanges()); + } + + if (cmdLineOptions.replay) { + optionParser = StringOptionParser(*cmdLineOptions.replay); + + auto dur = optionParser.parseDuration(StringOptionParser::FieldIs::kOptional); + + auto &cmd = _commands.emplace_back(CoincenterCommandType::kReplay) + .setReplayOptions(cmdLineOptions.computeReplayOptions(dur)) + .setExchangeNames(optionParser.parseExchanges()); + + if (!cmdLineOptions.market.empty()) { + cmd.setMarket(Market(cmdLineOptions.market)); + } + } + + if (cmdLineOptions.replayMarkets) { + optionParser = StringOptionParser(*cmdLineOptions.replayMarkets); + + TimeWindow timeWindow; + auto dur = optionParser.parseDuration(StringOptionParser::FieldIs::kOptional); + auto nowTime = Clock::now(); + if (dur == kUndefinedDuration) { + timeWindow = TimeWindow(TimePoint{}, nowTime); + } else { + timeWindow = TimeWindow(nowTime - dur, nowTime); + } + + _commands.emplace_back(CoincenterCommandType::kReplayMarkets) + .setReplayOptions( + ReplayOptions(timeWindow, cmdLineOptions.algorithmNames, ReplayOptions::ReplayMode::kValidateOnly)) + .setExchangeNames(optionParser.parseExchanges()); + } + optionParser.checkEndParsing(); // No more option part should be remaining } diff --git a/src/engine/src/coincenterinfo_create.cpp b/src/engine/src/coincenterinfo_create.cpp index a15663ec..225f734b 100644 --- a/src/engine/src/coincenterinfo_create.cpp +++ b/src/engine/src/coincenterinfo_create.cpp @@ -4,6 +4,7 @@ #include #include "apioutputtype.hpp" +#include "automation-config.hpp" #include "cct_json.hpp" #include "cct_string.hpp" #include "coincenterinfo.hpp" @@ -19,6 +20,7 @@ #include "runmodes.hpp" #include "stringoptionparser.hpp" #include "timedef.hpp" +#include "trading-config.hpp" namespace cct { @@ -70,8 +72,21 @@ CoincenterInfo CoincenterInfo_Create(std::string_view programName, const Coincen RequestsConfig requestsConfig( generalConfigData.at("requests").at("concurrency").at("nbMaxParallelRequests").get()); - GeneralConfig generalConfig(std::move(loggingInfo), std::move(requestsConfig), fiatConversionQueryRate, - apiOutputType); + const auto &automationJsonPart = generalConfigData.at("trading").at("automation"); + const auto &deserializationJsonPart = automationJsonPart.at("deserialization"); + const auto &startingContextJsonPart = automationJsonPart.at("startingContext"); + + Duration loadChunkDuration = ParseDuration(deserializationJsonPart.at("loadChunkDuration").get()); + MonetaryAmount startBaseAmountEquivalent{ + startingContextJsonPart.at("startBaseAmountEquivalent").get()}; + MonetaryAmount startQuoteAmountEquivalent{ + startingContextJsonPart.at("startQuoteAmountEquivalent").get()}; + + AutomationConfig automationConfig(loadChunkDuration, startBaseAmountEquivalent, startQuoteAmountEquivalent); + TradingConfig tradingConfig(std::move(automationConfig)); + + GeneralConfig generalConfig(std::move(loggingInfo), std::move(requestsConfig), std::move(tradingConfig), + fiatConversionQueryRate, apiOutputType); const LoadConfiguration loadConfiguration(dataDir, LoadConfiguration::ExchangeConfigFileType::kProd); diff --git a/src/engine/src/coincenteroptions.cpp b/src/engine/src/coincenteroptions.cpp index 94f532ec..96488c43 100644 --- a/src/engine/src/coincenteroptions.cpp +++ b/src/engine/src/coincenteroptions.cpp @@ -13,7 +13,10 @@ #include "monetaryamount.hpp" #include "priceoptions.hpp" #include "priceoptionsdef.hpp" +#include "replay-options.hpp" #include "ssl_sha.hpp" +#include "time-window.hpp" +#include "timedef.hpp" #include "tradedefinitions.hpp" #include "tradeoptions.hpp" #include "withdrawoptions.hpp" @@ -37,6 +40,9 @@ std::ostream& CoincenterCmdLineOptions::PrintVersion(std::string_view programNam os << "compiled with " << CCT_COMPILER_VERSION << " on " << __DATE__ << " at " << __TIME__ << '\n'; os << " " << GetCurlVersionInfo() << '\n'; os << " " << ssl::GetOpenSSLVersion() << '\n'; +#ifdef CCT_PROTOBUF_VERSION + os << " " << "protobuf " << CCT_PROTOBUF_VERSION << '\n'; +#endif return os; } @@ -132,6 +138,31 @@ WithdrawOptions CoincenterCmdLineOptions::computeWithdrawOptions() const { return {withdrawRefreshTime, withdrawSyncPolicy, mode}; } +ReplayOptions CoincenterCmdLineOptions::computeReplayOptions(Duration dur) const { + if (validate && validateOnly) { + throw invalid_argument("--validate and --validate-only cannot be specified simultaneously"); + } + + ReplayOptions::ReplayMode replayMode; + if (validateOnly) { + replayMode = ReplayOptions::ReplayMode::kValidateOnly; + } else if (validate) { + replayMode = ReplayOptions::ReplayMode::kCheckedLaunchAlgorithm; + } else { + replayMode = ReplayOptions::ReplayMode::kUncheckedLaunchAlgorithm; + } + + TimeWindow timeWindow; + const auto nowTime = Clock::now(); + if (dur == kUndefinedDuration) { + timeWindow = TimeWindow(TimePoint{}, nowTime); + } else { + timeWindow = TimeWindow(nowTime - dur, nowTime); + } + + return ReplayOptions(timeWindow, algorithmNames, replayMode); +} + std::pair CoincenterCmdLineOptions::getTradeArgStr() const { if (!tradeStrategy.empty() && !tradePrice.empty()) { throw invalid_argument("Trade price and trade strategy cannot be set together"); diff --git a/src/engine/src/exchangesorchestrator.cpp b/src/engine/src/exchangesorchestrator.cpp index 7132175f..b1347c74 100644 --- a/src/engine/src/exchangesorchestrator.cpp +++ b/src/engine/src/exchangesorchestrator.cpp @@ -31,13 +31,16 @@ #include "exchangepublicapi.hpp" #include "exchangepublicapitypes.hpp" #include "exchangeretriever.hpp" +#include "market-trader-engine.hpp" #include "market.hpp" #include "monetaryamount.hpp" #include "monetaryamountbycurrencyset.hpp" #include "ordersconstraints.hpp" #include "queryresulttypes.hpp" +#include "replay-options.hpp" #include "requestsconfig.hpp" #include "threadpool.hpp" +#include "trade-range-stats.hpp" #include "tradedamounts.hpp" #include "tradeoptions.hpp" #include "traderesult.hpp" @@ -170,7 +173,7 @@ MarketOrderBookConversionRates ExchangesOrchestrator::getMarketOrderBooks(Market if (!optConversionRate && !equiCurrencyCode.isNeutral()) { log::warn("Unable to convert {} into {} on {}", mk.quote(), equiCurrencyCode, exchange->name()); } - return std::make_tuple(exchange->name(), exchange->queryOrderBook(mk, actualDepth), optConversionRate); + return std::make_tuple(exchange->name(), exchange->getOrderBook(mk, actualDepth), optConversionRate); }; _threadPool.parallelTransform(selectedExchanges.begin(), selectedExchanges.end(), ret.begin(), marketOrderBooksFunc); return ret; @@ -941,7 +944,7 @@ TradesPerExchange ExchangesOrchestrator::getLastTradesPerExchange(Market mk, Exc TradesPerExchange ret(selectedExchanges.size()); _threadPool.parallelTransform( selectedExchanges.begin(), selectedExchanges.end(), ret.begin(), [mk, nbLastTrades](Exchange *exchange) { - return std::make_pair(static_cast(exchange), exchange->queryLastTrades(mk, nbLastTrades)); + return std::make_pair(static_cast(exchange), exchange->getLastTrades(mk, nbLastTrades)); }); return ret; @@ -958,4 +961,116 @@ MonetaryAmountPerExchange ExchangesOrchestrator::getLastPricePerExchange(Market return lastPricePerExchange; } +MarketDataPerExchange ExchangesOrchestrator::getMarketDataPerExchange(std::span marketPerPublicExchange, + ExchangeNameSpan exchangeNames) { + UniquePublicSelectedExchanges selectedExchanges = _exchangeRetriever.selectOneAccount(exchangeNames); + + std::array isMarketTradable; + + _threadPool.parallelTransform(selectedExchanges.begin(), selectedExchanges.end(), isMarketTradable.begin(), + [&marketPerPublicExchange](Exchange *exchange) { + Market market = marketPerPublicExchange[exchange->publicExchangePos()]; + return market.isDefined() && exchange->queryTradableMarkets().contains(market); + }); + + FilterVector(selectedExchanges, isMarketTradable); + + MarketDataPerExchange ret(selectedExchanges.size()); + _threadPool.parallelTransform( + selectedExchanges.begin(), selectedExchanges.end(), ret.begin(), [&marketPerPublicExchange](Exchange *exchange) { + if (!exchange->exchangeConfig().withMarketDataSerialization()) { + log::warn("Calling market-data on {} with data serialization disabled", exchange->name()); + } + // Call order book and last trades sequentially for this exchange + Market market = marketPerPublicExchange[exchange->publicExchangePos()]; + return std::make_pair(exchange, + std::make_pair(exchange->getOrderBook(market), exchange->getLastTrades(market))); + }); + return ret; +} + +MarketTimestampSetsPerExchange ExchangesOrchestrator::pullAvailableMarketsForReplay(TimeWindow timeWindow, + ExchangeNameSpan exchangeNames) { + log::info("Query available markets for replay from {} within {}", ConstructAccumulatedExchangeNames(exchangeNames), + timeWindow); + UniquePublicSelectedExchanges selectedExchanges = _exchangeRetriever.selectOneAccount(exchangeNames); + MarketTimestampSetsPerExchange marketTimestampSetsPerExchange(selectedExchanges.size()); + _threadPool.parallelTransform(selectedExchanges.begin(), selectedExchanges.end(), + marketTimestampSetsPerExchange.begin(), [timeWindow](Exchange *exchange) { + return std::make_pair( + exchange, + MarketTimestampSets{exchange->apiPublic().pullMarketOrderBooksMarkets(timeWindow), + exchange->apiPublic().pullTradeMarkets(timeWindow)}); + }); + return marketTimestampSetsPerExchange; +} + +MarketTradeRangeStatsPerExchange ExchangesOrchestrator::traderConsumeRange( + const ReplayOptions &replayOptions, TimeWindow subTimeWindow, std::span marketTraderEngines, + ExchangeNameSpan exchangeNames) { + UniquePublicSelectedExchanges selectedExchanges = _exchangeRetriever.selectOneAccount(exchangeNames); + + MarketTradeRangeStatsPerExchange tradeRangeResultsPerExchange(selectedExchanges.size()); + + _threadPool.parallelTransform( + selectedExchanges.begin(), selectedExchanges.end(), marketTraderEngines.begin(), + tradeRangeResultsPerExchange.begin(), + [subTimeWindow, &replayOptions](Exchange *exchange, MarketTraderEngine &marketTraderEngine) { + Market market = marketTraderEngine.market(); + auto &apiPublic = exchange->apiPublic(); + auto marketOrderBooks = apiPublic.pullMarketOrderBooksForReplay(market, subTimeWindow); + auto publicTrades = apiPublic.pullTradesForReplay(market, subTimeWindow); + + TradeRangeStats tradeRangeStats; + + switch (replayOptions.replayMode()) { + case ReplayOptions::ReplayMode::kValidateOnly: + tradeRangeStats = marketTraderEngine.validateRange(std::move(marketOrderBooks), std::move(publicTrades)); + break; + case ReplayOptions::ReplayMode::kCheckedLaunchAlgorithm: + tradeRangeStats = marketTraderEngine.validateRange(marketOrderBooks, publicTrades); + marketTraderEngine.tradeRange(std::move(marketOrderBooks), std::move(publicTrades)); + break; + case ReplayOptions::ReplayMode::kUncheckedLaunchAlgorithm: + tradeRangeStats = marketTraderEngine.tradeRange(std::move(marketOrderBooks), std::move(publicTrades)); + break; + default: + break; + } + + return std::make_pair(exchange, std::move(tradeRangeStats)); + }); + + return tradeRangeResultsPerExchange; +} + +MarketTradingGlobalResultPerExchange ExchangesOrchestrator::getMarketTraderResultPerExchange( + std::span marketTraderEngines, MarketTradeRangeStatsPerExchange &&tradeRangeStatsPerExchange, + ExchangeNameSpan exchangeNames) { + UniquePublicSelectedExchanges selectedExchanges = _exchangeRetriever.selectOneAccount(exchangeNames); + + if (selectedExchanges.size() != tradeRangeStatsPerExchange.size()) { + throw exception("Inconsistent selected exchange sizes"); + } + + MarketTradingResultPerExchange marketTradingResultPerExchange(selectedExchanges.size()); + + _threadPool.parallelTransform(selectedExchanges.begin(), selectedExchanges.end(), marketTraderEngines.begin(), + marketTradingResultPerExchange.begin(), + [](const Exchange *exchange, MarketTraderEngine &marketTraderEngine) { + return std::make_pair(exchange, marketTraderEngine.finalizeAndComputeResult()); + }); + + MarketTradingGlobalResultPerExchange marketTradingGlobalResultPerExchange(selectedExchanges.size()); + std::transform(marketTradingResultPerExchange.begin(), marketTradingResultPerExchange.end(), + tradeRangeStatsPerExchange.begin(), marketTradingGlobalResultPerExchange.begin(), + [](auto &exchangeMarketTradingResult, auto &exchangeTradeRangeStats) { + return std::make_pair(exchangeMarketTradingResult.first, + MarketTradingGlobalResult{std::move(exchangeMarketTradingResult.second), + std::move(exchangeTradeRangeStats.second)}); + }); + + return marketTradingGlobalResultPerExchange; +} + } // namespace cct diff --git a/src/engine/src/query-result-type-helpers.cpp b/src/engine/src/query-result-type-helpers.cpp new file mode 100644 index 00000000..866aec2f --- /dev/null +++ b/src/engine/src/query-result-type-helpers.cpp @@ -0,0 +1,34 @@ +#include "query-result-type-helpers.hpp" + +#include +#include + +#include "exchangepublicapitypes.hpp" +#include "market-timestamp-set.hpp" +#include "market.hpp" +#include "queryresulttypes.hpp" + +namespace cct { + +bool ContainsMarket(Market market, const MarketTimestampSet &marketTimestampSet) { + auto it = std::ranges::partition_point( + marketTimestampSet, [market](const auto &marketTimestamp) { return marketTimestamp.market < market; }); + return it != marketTimestampSet.end() && it->market == market; +} + +bool ContainsMarket(Market market, const MarketTimestampSets &marketTimestampSets) { + return ContainsMarket(market, marketTimestampSets.orderBooksMarkets) || + ContainsMarket(market, marketTimestampSets.tradesMarkets); +} + +MarketSet ComputeAllMarkets(const MarketTimestampSetsPerExchange &marketTimestampSetsPerExchange) { + MarketSet allMarkets; + for (const auto &[_, marketTimestamps] : marketTimestampSetsPerExchange) { + std::ranges::transform(marketTimestamps.orderBooksMarkets, std::inserter(allMarkets, allMarkets.end()), + [](const auto &marketTimestamp) { return marketTimestamp.market; }); + std::ranges::transform(marketTimestamps.tradesMarkets, std::inserter(allMarkets, allMarkets.end()), + [](const auto &marketTimestamp) { return marketTimestamp.market; }); + } + return allMarkets; +} +} // namespace cct \ No newline at end of file diff --git a/src/engine/src/queryresultprinter.cpp b/src/engine/src/queryresultprinter.cpp index 67bd223b..678ff1d2 100644 --- a/src/engine/src/queryresultprinter.cpp +++ b/src/engine/src/queryresultprinter.cpp @@ -27,6 +27,7 @@ #include "exchange.hpp" #include "file.hpp" #include "logginginfo.hpp" +#include "market-timestamp.hpp" #include "market.hpp" #include "marketorderbook.hpp" #include "monetaryamount.hpp" @@ -36,9 +37,11 @@ #include "priceoptions.hpp" #include "priceoptionsdef.hpp" #include "publictrade.hpp" +#include "query-result-type-helpers.hpp" #include "queryresulttypes.hpp" #include "simpletable.hpp" #include "stringhelpers.hpp" +#include "time-window.hpp" #include "timestring.hpp" #include "tradedamounts.hpp" #include "tradedefinitions.hpp" @@ -126,6 +129,47 @@ json MarketsJson(CurrencyCode cur1, CurrencyCode cur2, const MarketsPerExchange return ToJson(CoincenterCommandType::kMarkets, std::move(in), std::move(out)); } +json MarketsForReplayJson(TimeWindow timeWindow, const MarketTimestampSetsPerExchange &marketTimestampSetsPerExchange) { + json in; + json inOpt = json::object(); + if (timeWindow != TimeWindow{}) { + inOpt.emplace("timeWindow", timeWindow.str()); + } + in.emplace("opt", std::move(inOpt)); + + json out = json::object(); + for (const auto &[e, marketTimestampSets] : marketTimestampSetsPerExchange) { + json orderBookMarketsPerExchange; + for (const MarketTimestamp &marketTimestamp : marketTimestampSets.orderBooksMarkets) { + json marketTimestampJson; + + marketTimestampJson.emplace("market", marketTimestamp.market.str()); + marketTimestampJson.emplace("lastTimestamp", ToString(marketTimestamp.timePoint)); + + orderBookMarketsPerExchange.emplace_back(std::move(marketTimestampJson)); + } + + json tradesMarketsPerExchange; + for (const MarketTimestamp &marketTimestamp : marketTimestampSets.tradesMarkets) { + json marketTimestampJson; + + marketTimestampJson.emplace("market", marketTimestamp.market.str()); + marketTimestampJson.emplace("lastTimestamp", ToString(marketTimestamp.timePoint)); + + tradesMarketsPerExchange.emplace_back(std::move(marketTimestampJson)); + } + + json exchangePart; + + exchangePart.emplace("orderBooks", std::move(orderBookMarketsPerExchange)); + exchangePart.emplace("trades", std::move(tradesMarketsPerExchange)); + + out.emplace(e->name(), std::move(exchangePart)); + } + + return ToJson(CoincenterCommandType::kReplayMarkets, std::move(in), std::move(out)); +} + json TickerInformationJson(const ExchangeTickerMaps &exchangeTickerMaps) { json in; json out = json::object(); @@ -722,6 +766,57 @@ json DustSweeperJson(const TradedAmountsVectorWithFinalAmountPerExchange &traded return ToJson(CoincenterCommandType::kDustSweeper, std::move(in), std::move(out)); } +json MarketTradingResultsJson(TimeWindow timeWindow, + const MarketTradingGlobalResultPerExchange &marketTradingResultPerExchange, + CoincenterCommandType commandType) { + json in; + json inOpt; + inOpt.emplace("time-window", timeWindow.str()); + in.emplace("opt", std::move(inOpt)); + + json out = json::object(); + + for (const auto &[exchangePtr, marketGlobalTradingResult] : marketTradingResultPerExchange) { + const auto &marketTradingResult = marketGlobalTradingResult.result; + const auto &stats = marketGlobalTradingResult.stats; + + json startAmounts; + startAmounts.emplace("base", marketTradingResult.startBaseAmount().str()); + startAmounts.emplace("quote", marketTradingResult.startQuoteAmount().str()); + + json orderBookStats; + orderBookStats.emplace("nb-successful", stats.marketOrderBookStats.nbSuccessful); + orderBookStats.emplace("nb-error", stats.marketOrderBookStats.nbError); + + json tradeStats; + tradeStats.emplace("nb-successful", stats.publicTradeStats.nbSuccessful); + tradeStats.emplace("nb-error", stats.publicTradeStats.nbError); + + json jsonStats; + jsonStats.emplace("order-books", std::move(orderBookStats)); + jsonStats.emplace("trades", std::move(tradeStats)); + + json marketTradingResultJson; + marketTradingResultJson.emplace("algorithm", marketTradingResult.algorithmName()); + marketTradingResultJson.emplace("market", marketTradingResult.market().str()); + marketTradingResultJson.emplace("start-amounts", std::move(startAmounts)); + marketTradingResultJson.emplace("profit-and-loss", marketTradingResult.quoteAmountDelta().str()); + marketTradingResultJson.emplace("stats", std::move(jsonStats)); + + json closedOrdersArray = json::array_t(); + + for (const ClosedOrder &closedOrder : marketTradingResult.matchedOrders()) { + closedOrdersArray.push_back(OrderJson(closedOrder)); + } + + marketTradingResultJson.emplace("matched-orders", std::move(closedOrdersArray)); + + out.emplace(exchangePtr->name(), std::move(marketTradingResultJson)); + } + + return ToJson(commandType, std::move(in), std::move(out)); +} + template void RemoveDuplicates(VecType &vec) { std::ranges::sort(vec); @@ -1452,6 +1547,123 @@ void QueryResultPrinter::printDustSweeper( logActivity(CoincenterCommandType::kDustSweeper, jsonData); } +void QueryResultPrinter::printMarketsForReplay(TimeWindow timeWindow, + const MarketTimestampSetsPerExchange &marketTimestampSetsPerExchange) { + json jsonData = MarketsForReplayJson(timeWindow, marketTimestampSetsPerExchange); + switch (_apiOutputType) { + case ApiOutputType::kFormattedTable: { + MarketSet allMarkets = ComputeAllMarkets(marketTimestampSetsPerExchange); + + SimpleTable table; + table.reserve(allMarkets.size() + 1U); + table.emplace_back("Markets", "Last order books timestamp", "Last trades timestamp"); + + for (const Market market : allMarkets) { + table::Cell orderBookCell; + table::Cell tradesCell; + for (const auto &[e, marketTimestamps] : marketTimestampSetsPerExchange) { + const auto &orderBooksMarkets = marketTimestamps.orderBooksMarkets; + const auto &tradesMarkets = marketTimestamps.tradesMarkets; + const auto marketPartitionPred = [market](const auto &marketTimestamp) { + return marketTimestamp.market < market; + }; + const auto orderBooksIt = std::ranges::partition_point(orderBooksMarkets, marketPartitionPred); + const auto tradesIt = std::ranges::partition_point(tradesMarkets, marketPartitionPred); + + if (orderBooksIt != orderBooksMarkets.end() && orderBooksIt->market == market) { + string str = ToString(orderBooksIt->timePoint); + str.append(" @ "); + str.append(e->name()); + + orderBookCell.emplace_back(std::move(str)); + } + + if (tradesIt != tradesMarkets.end() && tradesIt->market == market) { + string str = ToString(tradesIt->timePoint); + str.append(" @ "); + str.append(e->name()); + + tradesCell.emplace_back(std::move(str)); + } + } + + table.emplace_back(market.str(), std::move(orderBookCell), std::move(tradesCell)); + } + printTable(table); + break; + } + case ApiOutputType::kJson: + printJson(jsonData); + break; + case ApiOutputType::kNoPrint: + break; + } + logActivity(CoincenterCommandType::kReplayMarkets, jsonData); +} + +void QueryResultPrinter::printMarketTradingResults( + TimeWindow timeWindow, const MarketTradingGlobalResultPerExchange &marketTradingResultPerExchange, + CoincenterCommandType commandType) const { + json jsonData = MarketTradingResultsJson(timeWindow, marketTradingResultPerExchange, commandType); + switch (_apiOutputType) { + case ApiOutputType::kFormattedTable: { + SimpleTable table; + table.reserve(1U + marketTradingResultPerExchange.size()); + table.emplace_back("Exchange", "Time window", "Market", "Algorithm", "Start amounts", "Profit / Loss", + "Matched orders", "Stats"); + for (const auto &[exchangePtr, marketGlobalTradingResults] : marketTradingResultPerExchange) { + const auto &marketTradingResults = marketGlobalTradingResults.result; + const auto &stats = marketGlobalTradingResults.stats; + + table::Cell trades; + for (const ClosedOrder &closedOrder : marketTradingResults.matchedOrders()) { + string orderStr = closedOrder.placedTimeStr(); + orderStr.append(" - "); + orderStr.append(closedOrder.sideStr()); + orderStr.append(" - "); + orderStr.append(closedOrder.matchedVolume().str()); + orderStr.append(" @ "); + orderStr.append(closedOrder.price().str()); + trades.emplace_back(std::move(orderStr)); + } + + string orderBookStats("order books: "); + orderBookStats.append(std::string_view(ToCharVector(stats.marketOrderBookStats.nbSuccessful))); + orderBookStats.append(" OK"); + if (stats.marketOrderBookStats.nbError != 0) { + orderBookStats.append(", "); + orderBookStats.append(std::string_view(ToCharVector(stats.marketOrderBookStats.nbError))); + orderBookStats.append(" KO"); + } + + string tradesStats("trades: "); + tradesStats.append(std::string_view(ToCharVector(stats.publicTradeStats.nbSuccessful))); + tradesStats.append(" OK"); + if (stats.publicTradeStats.nbError != 0) { + tradesStats.append(", "); + tradesStats.append(std::string_view(ToCharVector(stats.publicTradeStats.nbError))); + tradesStats.append(" KO"); + } + + table.emplace_back( + exchangePtr->name(), table::Cell{ToString(timeWindow.from()), ToString(timeWindow.to())}, + marketTradingResults.market().str(), marketTradingResults.algorithmName(), + table::Cell{marketTradingResults.startBaseAmount().str(), marketTradingResults.startQuoteAmount().str()}, + marketTradingResults.quoteAmountDelta().str(), std::move(trades), + table::Cell{std::move(orderBookStats), std::move(tradesStats)}); + } + printTable(table); + break; + } + case ApiOutputType::kJson: + printJson(jsonData); + break; + case ApiOutputType::kNoPrint: + break; + } + logActivity(commandType, jsonData); +} + void QueryResultPrinter::printTable(const SimpleTable &table) const { std::ostringstream ss; std::ostream &os = _pOs != nullptr ? *_pOs : ss; diff --git a/src/engine/src/replay-algorithm-name-iterator.cpp b/src/engine/src/replay-algorithm-name-iterator.cpp new file mode 100644 index 00000000..1a35a3a8 --- /dev/null +++ b/src/engine/src/replay-algorithm-name-iterator.cpp @@ -0,0 +1,63 @@ +#include "replay-algorithm-name-iterator.hpp" + +#include +#include +#include + +#include "cct_exception.hpp" + +namespace cct { + +namespace { +constexpr std::string_view kAlgorithmNameSeparator = ","; + +auto FindNextSeparatorPos(std::string_view str, std::string_view::size_type pos = 0) { + pos = str.find(kAlgorithmNameSeparator, pos); + if (pos == std::string_view::npos) { + pos = str.length(); + } + return pos; +} +} // namespace + +ReplayAlgorithmNameIterator::ReplayAlgorithmNameIterator(std::string_view algorithmNames, + std::span allAlgorithms) + : _allAlgorithms(allAlgorithms), + _algorithmNames(algorithmNames), + _begPos(0), + _endPos(FindNextSeparatorPos(_algorithmNames)) { + if (std::ranges::any_of(allAlgorithms, [](const auto algName) { + return algName.find(kAlgorithmNameSeparator) != std::string_view::npos; + })) { + throw exception("Algorithm names cannot contain '{}' as it's used as a separator", kAlgorithmNameSeparator); + } +} + +bool ReplayAlgorithmNameIterator::hasNext() const { + using PosT = decltype(_begPos); + + if (_algorithmNames.empty()) { + return _begPos < static_cast(_allAlgorithms.size()); + } + + return _begPos != static_cast(_algorithmNames.length()); +} + +std::string_view ReplayAlgorithmNameIterator::next() { + if (_algorithmNames.empty()) { + return _allAlgorithms[_begPos++]; + } + + std::string_view nextAlgorithmName(_algorithmNames.begin() + _begPos, _algorithmNames.begin() + _endPos); + + if (_endPos == static_cast(_algorithmNames.length())) { + _begPos = _endPos; + } else { + _begPos = _endPos + kAlgorithmNameSeparator.length(); + _endPos = FindNextSeparatorPos(_algorithmNames, _begPos); + } + + return nextAlgorithmName; +} + +} // namespace cct \ No newline at end of file diff --git a/src/engine/src/replay-options.cpp b/src/engine/src/replay-options.cpp new file mode 100644 index 00000000..7eb728c2 --- /dev/null +++ b/src/engine/src/replay-options.cpp @@ -0,0 +1,20 @@ +#include "replay-options.hpp" + +#include + +#include "dummy-market-trader.hpp" +#include "time-window.hpp" + +namespace cct { + +ReplayOptions::ReplayOptions(TimeWindow timeWindow, std::string_view algorithmNames, ReplayMode replayMode) + : _timeWindow(timeWindow), _algorithmNames(algorithmNames), _replayMode(replayMode) {} + +std::string_view ReplayOptions::algorithmNames() const { + if (_replayMode == ReplayMode::kValidateOnly) { + return DummyMarketTrader::kName; + } + return _algorithmNames; +} + +} // namespace cct \ No newline at end of file diff --git a/src/engine/src/stringoptionparser.cpp b/src/engine/src/stringoptionparser.cpp index 11502459..6435837d 100644 --- a/src/engine/src/stringoptionparser.cpp +++ b/src/engine/src/stringoptionparser.cpp @@ -9,10 +9,12 @@ #include "cct_string.hpp" #include "cct_vector.hpp" #include "currencycode.hpp" +#include "durationstring.hpp" #include "exchange-names.hpp" #include "exchangename.hpp" #include "market.hpp" #include "monetaryamount.hpp" +#include "timedef.hpp" namespace cct { @@ -38,6 +40,23 @@ CurrencyCode StringOptionParser::parseCurrency(FieldIs fieldIs, char delimiter) return {}; } +Duration StringOptionParser::parseDuration(FieldIs fieldIs) { + auto dur = kUndefinedDuration; + const std::string_view currentToken(_opt.begin() + _pos, _opt.end()); + const auto durationLen = DurationLen(currentToken); + if (durationLen > 0) { + const std::string_view durationStr(_opt.data() + _pos, static_cast(durationLen)); + + dur = ParseDuration(durationStr); + } else if (fieldIs == FieldIs::kMandatory) { + throw invalid_argument("Expected a valid duration in '{}'", currentToken); + } + + _pos += durationLen; + + return dur; +} + // At the end of the market, either the end of the string or a comma is expected. Market StringOptionParser::parseMarket(FieldIs fieldIs, char delimiter) { const auto oldPos = _pos; diff --git a/src/engine/test/queryresultprinter_public_test.cpp b/src/engine/test/queryresultprinter_public_test.cpp index 57bbc12e..6ba960df 100644 --- a/src/engine/test/queryresultprinter_public_test.cpp +++ b/src/engine/test/queryresultprinter_public_test.cpp @@ -9,15 +9,21 @@ #include "currencycode.hpp" #include "currencyexchange.hpp" #include "currencyexchangeflatset.hpp" +#include "exchangeprivateapitypes.hpp" #include "exchangepublicapitypes.hpp" +#include "market-trading-global-result.hpp" +#include "market-trading-result.hpp" #include "market.hpp" #include "marketorderbook.hpp" #include "monetaryamount.hpp" #include "monetaryamountbycurrencyset.hpp" +#include "public-trade-vector.hpp" #include "publictrade.hpp" #include "queryresultprinter.hpp" #include "queryresultprinter_base_test.hpp" #include "queryresulttypes.hpp" +#include "time-window.hpp" +#include "trade-range-stats.hpp" #include "tradeside.hpp" namespace cct { @@ -243,7 +249,6 @@ TEST_F(QueryResultPrinterMarketsTest, FormattedTableNoCurrency) { | huobi | XRP-EUR | +----------+---------+ )"; - expectStr(kExpected); } @@ -262,7 +267,6 @@ TEST_F(QueryResultPrinterMarketsTest, FormattedTableOneCurrency) { | huobi | XRP-EUR | +----------+------------------+ )"; - expectStr(kExpected); } @@ -279,7 +283,6 @@ TEST_F(QueryResultPrinterMarketsTest, FormattedTableTwoCurrencies) { | huobi | XRP-EUR | +----------+----------------------+ )"; - expectStr(kExpected); } @@ -1176,4 +1179,303 @@ TEST_F(QueryResultPrinterLastPriceTest, NoPrint) { expectNoStr(); } +class QueryResultPrinterReplayBaseTest : public QueryResultPrinterTest { + protected: + Market market1{"ETH", "KRW"}; + Market market2{"BTC", "USD"}; + Market market3{"SHIB", "USDT"}; + Market market4{"SOL", "BTC"}; + Market market5{"SOL", "ETH"}; + Market market6{"ETH", "BTC"}; + Market market7{"DOGE", "CAD"}; + + TimePoint tp1{milliseconds{std::numeric_limits::max() / 10000000}}; + TimePoint tp2{milliseconds{std::numeric_limits::max() / 9900000}}; + TimePoint tp3{milliseconds{std::numeric_limits::max() / 9800000}}; + TimePoint tp4{milliseconds{std::numeric_limits::max() / 9600000}}; + TimePoint tp5{milliseconds{std::numeric_limits::max() / 9500000}}; + + TimeWindow timeWindow{tp1, tp5}; +}; + +class QueryResultPrinterReplayMarketsTest : public QueryResultPrinterReplayBaseTest { + protected: + MarketTimestampSetsPerExchange marketTimestampSetsPerExchange{ + {&exchange1, + MarketTimestampSets{MarketTimestampSet{MarketTimestamp{market1, tp1}, MarketTimestamp{market2, tp2}, + MarketTimestamp{market3, tp3}}, + MarketTimestampSet{MarketTimestamp{market1, tp1}, MarketTimestamp{market2, tp1}}}}, + {&exchange2, MarketTimestampSets{MarketTimestampSet{MarketTimestamp{market2, tp4}, MarketTimestamp{market4, tp5}}, + MarketTimestampSet{MarketTimestamp{market6, tp1}}}}, + {&exchange3, MarketTimestampSets{MarketTimestampSet{}, MarketTimestampSet{MarketTimestamp{market1, tp1}, + MarketTimestamp{market7, tp4}}}}}; +}; + +TEST_F(QueryResultPrinterReplayMarketsTest, FormattedTable) { + basicQueryResultPrinter(ApiOutputType::kFormattedTable) + .printMarketsForReplay(timeWindow, marketTimestampSetsPerExchange); + static constexpr std::string_view kExpected = R"( ++-----------+--------------------------------+--------------------------------+ +| Markets | Last order books timestamp | Last trades timestamp | ++-----------+--------------------------------+--------------------------------+ +| BTC-USD | 1999-07-11T00:42:21Z @ binance | 1999-03-25T04:46:43Z @ binance | +| | 2000-06-11T23:58:40Z @ bithumb | | +|~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~| +| DOGE-CAD | | 2000-06-11T23:58:40Z @ huobi | +| ETH-BTC | | 1999-03-25T04:46:43Z @ bithumb | +|~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~| +| ETH-KRW | 1999-03-25T04:46:43Z @ binance | 1999-03-25T04:46:43Z @ binance | +| | | 1999-03-25T04:46:43Z @ huobi | +|~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~| +| SHIB-USDT | 1999-10-29T01:26:51Z @ binance | | +| SOL-BTC | 2000-10-07T01:14:27Z @ bithumb | | ++-----------+--------------------------------+--------------------------------+ +)"; + expectStr(kExpected); +} + +TEST_F(QueryResultPrinterReplayMarketsTest, EmptyJson) { + basicQueryResultPrinter(ApiOutputType::kJson).printMarketsForReplay(timeWindow, MarketTimestampSetsPerExchange{}); + static constexpr std::string_view kExpected = R"json( +{ + "in": { + "opt": { + "timeWindow": "[1999-03-25 04:46:43 -> 2000-10-07 01:14:27)" + }, + "req": "ReplayMarkets" + }, + "out": {} +})json"; + expectJson(kExpected); +} + +TEST_F(QueryResultPrinterReplayMarketsTest, Json) { + basicQueryResultPrinter(ApiOutputType::kJson).printMarketsForReplay(timeWindow, marketTimestampSetsPerExchange); + static constexpr std::string_view kExpected = R"json( +{ + "in": { + "opt": { + "timeWindow": "[1999-03-25 04:46:43 -> 2000-10-07 01:14:27)" + }, + "req": "ReplayMarkets" + }, + "out": { + "binance": { + "orderBooks": [ + { + "lastTimestamp": "1999-07-11T00:42:21Z", + "market": "BTC-USD" + }, + { + "lastTimestamp": "1999-03-25T04:46:43Z", + "market": "ETH-KRW" + }, + { + "lastTimestamp": "1999-10-29T01:26:51Z", + "market": "SHIB-USDT" + } + ], + "trades": [ + { + "lastTimestamp": "1999-03-25T04:46:43Z", + "market": "BTC-USD" + }, + { + "lastTimestamp": "1999-03-25T04:46:43Z", + "market": "ETH-KRW" + } + ] + }, + "bithumb": { + "orderBooks": [ + { + "lastTimestamp": "2000-06-11T23:58:40Z", + "market": "BTC-USD" + }, + { + "lastTimestamp": "2000-10-07T01:14:27Z", + "market": "SOL-BTC" + } + ], + "trades": [ + { + "lastTimestamp": "1999-03-25T04:46:43Z", + "market": "ETH-BTC" + } + ] + }, + "huobi": { + "orderBooks": null, + "trades": [ + { + "lastTimestamp": "2000-06-11T23:58:40Z", + "market": "DOGE-CAD" + }, + { + "lastTimestamp": "1999-03-25T04:46:43Z", + "market": "ETH-KRW" + } + ] + } + } +})json"; + expectJson(kExpected); +} + +TEST_F(QueryResultPrinterReplayMarketsTest, NoPrint) { + basicQueryResultPrinter(ApiOutputType::kNoPrint).printMarketsForReplay(timeWindow, marketTimestampSetsPerExchange); + expectNoStr(); +} + +class QueryResultPrinterReplayTest : public QueryResultPrinterReplayBaseTest { + protected: + ClosedOrder closedOrder1{"1", MonetaryAmount(15, "BTC", 1), MonetaryAmount(35000, "USDT"), tp1, tp1, TradeSide::kBuy}; + ClosedOrder closedOrder2{"2", MonetaryAmount(25, "BTC", 1), MonetaryAmount(45000, "USDT"), tp2, tp2, TradeSide::kBuy}; + ClosedOrder closedOrder3{"3", MonetaryAmount(5, "BTC", 2), MonetaryAmount(35000, "USDT"), tp3, tp4, TradeSide::kSell}; + ClosedOrder closedOrder4{ + "4", MonetaryAmount(17, "BTC", 1), MonetaryAmount(50000, "USDT"), tp3, tp4, TradeSide::kSell}; + ClosedOrder closedOrder5{ + "5", MonetaryAmount(36, "BTC", 3), MonetaryAmount(47899, "USDT"), tp4, tp5, TradeSide::kSell}; + + std::string_view algorithmName = "test-algo"; + MonetaryAmount startBaseAmount{1, "BTC"}; + MonetaryAmount startQuoteAmount{1000, "EUR"}; + + MarketTradingResult marketTradingResult1{algorithmName, startBaseAmount, startQuoteAmount, MonetaryAmount{0, "EUR"}, + ClosedOrderVector{}}; + MarketTradingResult marketTradingResult3{algorithmName, startBaseAmount, startQuoteAmount, MonetaryAmount{500, "EUR"}, + ClosedOrderVector{closedOrder1, closedOrder5}}; + MarketTradingResult marketTradingResult4{algorithmName, startBaseAmount, startQuoteAmount, MonetaryAmount{780, "EUR"}, + ClosedOrderVector{closedOrder2, closedOrder3, closedOrder4}}; + + TradeRangeStats tradeRangeStats1{TradeRangeResultsStats{42, 0}, TradeRangeResultsStats{3, 10}}; + TradeRangeStats tradeRangeStats3{TradeRangeResultsStats{500000, 2}, TradeRangeResultsStats{0, 0}}; + TradeRangeStats tradeRangeStats4{TradeRangeResultsStats{79009, 0}, TradeRangeResultsStats{1555555555, 45}}; + + MarketTradingGlobalResultPerExchange marketTradingResultPerExchange{ + {&exchange1, MarketTradingGlobalResult{marketTradingResult1, tradeRangeStats1}}, + {&exchange3, MarketTradingGlobalResult{marketTradingResult3, tradeRangeStats3}}, + {&exchange4, MarketTradingGlobalResult{marketTradingResult4, tradeRangeStats4}}}; + CoincenterCommandType commandType{CoincenterCommandType::kReplay}; +}; + +TEST_F(QueryResultPrinterReplayTest, FormattedTable) { + basicQueryResultPrinter(ApiOutputType::kFormattedTable) + .printMarketTradingResults(timeWindow, marketTradingResultPerExchange, commandType); + static constexpr std::string_view kExpected = R"( ++----------+----------------------+---------+-----------+---------------+---------------+------------------------------------------------------+------------------------------+ +| Exchange | Time window | Market | Algorithm | Start amounts | Profit / Loss | Matched orders | Stats | ++----------+----------------------+---------+-----------+---------------+---------------+------------------------------------------------------+------------------------------+ +| binance | 1999-03-25T04:46:43Z | BTC-EUR | test-algo | 1 BTC | 0 EUR | | order books: 42 OK | +| | 2000-10-07T01:14:27Z | | | 1000 EUR | | | trades: 3 OK, 10 KO | +|~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~|~~~~~~~~~|~~~~~~~~~~~|~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~| +| huobi | 1999-03-25T04:46:43Z | BTC-EUR | test-algo | 1 BTC | 500 EUR | 1999-03-25T04:46:43Z - Buy - 1.5 BTC @ 35000 USDT | order books: 500000 OK, 2 KO | +| | 2000-10-07T01:14:27Z | | | 1000 EUR | | 2000-06-11T23:58:40Z - Sell - 0.036 BTC @ 47899 USDT | trades: 0 OK | +|~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~|~~~~~~~~~|~~~~~~~~~~~|~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~| +| huobi | 1999-03-25T04:46:43Z | BTC-EUR | test-algo | 1 BTC | 780 EUR | 1999-07-11T00:42:21Z - Buy - 2.5 BTC @ 45000 USDT | order books: 79009 OK | +| | 2000-10-07T01:14:27Z | | | 1000 EUR | | 1999-10-29T01:26:51Z - Sell - 0.05 BTC @ 35000 USDT | trades: 1555555555 OK, 45 KO | +| | | | | | | 1999-10-29T01:26:51Z - Sell - 1.7 BTC @ 50000 USDT | | ++----------+----------------------+---------+-----------+---------------+---------------+------------------------------------------------------+------------------------------+ +)"; + expectStr(kExpected); +} + +TEST_F(QueryResultPrinterReplayTest, EmptyJson) { + basicQueryResultPrinter(ApiOutputType::kJson) + .printMarketTradingResults(timeWindow, MarketTradingGlobalResultPerExchange{}, commandType); + static constexpr std::string_view kExpected = R"json( +{ + "in": { + "opt": { + "time-window": "[1999-03-25 04:46:43 -> 2000-10-07 01:14:27)" + }, + "req": "Replay" + }, + "out": {} +})json"; + expectJson(kExpected); +} + +TEST_F(QueryResultPrinterReplayTest, Json) { + basicQueryResultPrinter(ApiOutputType::kJson) + .printMarketTradingResults(timeWindow, marketTradingResultPerExchange, commandType); + static constexpr std::string_view kExpected = R"json( +{ + "in": { + "opt": { + "time-window": "[1999-03-25 04:46:43 -> 2000-10-07 01:14:27)" + }, + "req": "Replay" + }, + "out": { + "binance": { + "algorithm": "test-algo", + "market": "BTC-EUR", + "matched-orders": [], + "profit-and-loss": "0 EUR", + "start-amounts": { + "base": "1 BTC", + "quote": "1000 EUR" + }, + "stats": { + "order-books": { + "nb-error": 0, + "nb-successful": 42 + }, + "trades": { + "nb-error": 10, + "nb-successful": 3 + } + } + }, + "huobi": { + "algorithm": "test-algo", + "market": "BTC-EUR", + "matched-orders": [ + { + "id": "1", + "matched": "1.5", + "matchedTime": "1999-03-25T04:46:43Z", + "pair": "BTC-USDT", + "placedTime": "1999-03-25T04:46:43Z", + "price": "35000", + "side": "Buy" + }, + { + "id": "5", + "matched": "0.036", + "matchedTime": "2000-10-07T01:14:27Z", + "pair": "BTC-USDT", + "placedTime": "2000-06-11T23:58:40Z", + "price": "47899", + "side": "Sell" + } + ], + "profit-and-loss": "500 EUR", + "start-amounts": { + "base": "1 BTC", + "quote": "1000 EUR" + }, + "stats": { + "order-books": { + "nb-error": 2, + "nb-successful": 500000 + }, + "trades": { + "nb-error": 0, + "nb-successful": 0 + } + } + } + } +})json"; + expectJson(kExpected); +} + +TEST_F(QueryResultPrinterReplayTest, NoPrint) { + basicQueryResultPrinter(ApiOutputType::kNoPrint) + .printMarketTradingResults(timeWindow, marketTradingResultPerExchange, commandType); + expectNoStr(); +} + } // namespace cct diff --git a/src/engine/test/replay-algorithm-name-iterator_test.cpp b/src/engine/test/replay-algorithm-name-iterator_test.cpp new file mode 100644 index 00000000..ee930a0b --- /dev/null +++ b/src/engine/test/replay-algorithm-name-iterator_test.cpp @@ -0,0 +1,106 @@ +#include "replay-algorithm-name-iterator.hpp" + +#include + +#include + +#include "cct_exception.hpp" + +namespace cct { +class ReplayAlgorithmNameIteratorTest : public ::testing::Test { + protected: + static constexpr std::string_view kInvalidAlgorithmNames[] = {"any", "so-what,"}; + static constexpr std::string_view kAlgorithmNames[] = {"any", "so-what", "angry", + "bird", "Jack", "a-more-complex algorithm Name"}; +}; + +TEST_F(ReplayAlgorithmNameIteratorTest, AlgorithmNamesValidity) { + EXPECT_THROW(ReplayAlgorithmNameIterator("", kInvalidAlgorithmNames), exception); + EXPECT_NO_THROW(ReplayAlgorithmNameIterator("", kAlgorithmNames)); +} + +TEST_F(ReplayAlgorithmNameIteratorTest, IteratorWithAll) { + ReplayAlgorithmNameIterator it("", kAlgorithmNames); + + int algorithmPos = 0; + while (it.hasNext()) { + auto next = it.next(); + + switch (algorithmPos) { + case 0: + [[fallthrough]]; + case 1: + [[fallthrough]]; + case 2: + [[fallthrough]]; + case 3: + [[fallthrough]]; + case 4: + [[fallthrough]]; + case 5: + EXPECT_EQ(next, kAlgorithmNames[algorithmPos]); + break; + default: + throw exception("Unexpected number of algorithm names"); + } + + ++algorithmPos; + } + + EXPECT_EQ(algorithmPos, 6); +} + +TEST_F(ReplayAlgorithmNameIteratorTest, IteratorWithUniqueAlgorithmSpecified) { + ReplayAlgorithmNameIterator it("so-What", kAlgorithmNames); + + int algorithmPos = 0; + while (it.hasNext()) { + auto next = it.next(); + + switch (algorithmPos) { + case 0: + EXPECT_EQ(next, "so-What"); + break; + default: + throw exception("Unexpected number of algorithm names"); + } + + ++algorithmPos; + } + + EXPECT_EQ(algorithmPos, 1); +} + +TEST_F(ReplayAlgorithmNameIteratorTest, IteratorWithSpecifiedList) { + ReplayAlgorithmNameIterator it("Jack,whatever,so-what,some-algorithmNameThatIsNotInAll,with spaces", kAlgorithmNames); + + int algorithmPos = 0; + while (it.hasNext()) { + auto next = it.next(); + + switch (algorithmPos) { + case 0: + EXPECT_EQ(next, "Jack"); + break; + case 1: + EXPECT_EQ(next, "whatever"); + break; + case 2: + EXPECT_EQ(next, "so-what"); + break; + case 3: + EXPECT_EQ(next, "some-algorithmNameThatIsNotInAll"); + break; + case 4: + EXPECT_EQ(next, "with spaces"); + break; + default: + throw exception("Unexpected number of algorithm names"); + } + + ++algorithmPos; + } + + EXPECT_EQ(algorithmPos, 5); +} +} // namespace cct \ No newline at end of file diff --git a/src/engine/test/stringoptionparser_test.cpp b/src/engine/test/stringoptionparser_test.cpp index e8869af4..4059145b 100644 --- a/src/engine/test/stringoptionparser_test.cpp +++ b/src/engine/test/stringoptionparser_test.cpp @@ -2,6 +2,7 @@ #include +#include #include #include "cct_invalid_argument_exception.hpp" @@ -12,6 +13,7 @@ #include "exchangename.hpp" #include "market.hpp" #include "monetaryamount.hpp" +#include "timedef.hpp" namespace cct { namespace { @@ -211,4 +213,26 @@ TEST(StringOptionParserTest, ExchangesNotLast) { EXPECT_NO_THROW(parser.checkEndParsing()); } +TEST(StringOptionParserTest, ParseDurationMandatory) { + StringOptionParser parser(" 45min83s,kraken,upbit"); + + EXPECT_EQ(parser.parseDuration(StringOptionParser::FieldIs::kMandatory), + std::chrono::minutes{45} + std::chrono::seconds{83}); + EXPECT_EQ(parser.parseExchanges(',', '\0'), ExchangeNames({ExchangeName("kraken"), ExchangeName("upbit")})); + + EXPECT_NO_THROW(parser.checkEndParsing()); +} + +TEST(StringOptionParserTest, ParseDurationOptional) { + StringOptionParser parser("binance,huobi_user1,34h 4500ms"); + + EXPECT_EQ(parser.parseDuration(StringOptionParser::FieldIs::kOptional), kUndefinedDuration); + EXPECT_EQ(parser.parseExchanges(',', '\0'), ExchangeNames({ExchangeName("binance"), ExchangeName("huobi", "user1")})); + + EXPECT_EQ(parser.parseDuration(StringOptionParser::FieldIs::kOptional), + std::chrono::hours{34} + std::chrono::milliseconds{4500}); + + EXPECT_NO_THROW(parser.checkEndParsing()); +} + } // namespace cct \ No newline at end of file diff --git a/src/objects/CMakeLists.txt b/src/objects/CMakeLists.txt index 7bacf80b..385bbe35 100644 --- a/src/objects/CMakeLists.txt +++ b/src/objects/CMakeLists.txt @@ -99,6 +99,15 @@ add_unit_test( CCT_DISABLE_SPDLOG ) +add_unit_test( + time-window_test + test/time-window_test.cpp + LIBRARIES + coincenter_objects + DEFINITIONS + CCT_DISABLE_SPDLOG +) + add_unit_test( wallet_test test/wallet_test.cpp diff --git a/src/objects/include/automation-config.hpp b/src/objects/include/automation-config.hpp new file mode 100644 index 00000000..669bb8dc --- /dev/null +++ b/src/objects/include/automation-config.hpp @@ -0,0 +1,30 @@ +#pragma once + +#include + +#include "monetaryamount.hpp" +#include "timedef.hpp" + +namespace cct { +class AutomationConfig { + public: + AutomationConfig() noexcept = default; + + AutomationConfig(Duration loadChunkDuration, MonetaryAmount startBaseAmountEquivalent, + MonetaryAmount startQuoteAmountEquivalent) + : _loadChunkDuration(loadChunkDuration), + _startBaseAmountEquivalent(startBaseAmountEquivalent), + _startQuoteAmountEquivalent(startQuoteAmountEquivalent) {} + + Duration loadChunkDuration() const { return _loadChunkDuration; } + + MonetaryAmount startBaseAmountEquivalent() const { return _startBaseAmountEquivalent; } + + MonetaryAmount startQuoteAmountEquivalent() const { return _startQuoteAmountEquivalent; } + + private: + Duration _loadChunkDuration = std::chrono::weeks(1); + MonetaryAmount _startBaseAmountEquivalent; + MonetaryAmount _startQuoteAmountEquivalent; +}; +} // namespace cct \ No newline at end of file diff --git a/src/objects/include/coincentercommandtype.hpp b/src/objects/include/coincentercommandtype.hpp index cc1002dc..49967dc6 100644 --- a/src/objects/include/coincentercommandtype.hpp +++ b/src/objects/include/coincentercommandtype.hpp @@ -31,6 +31,10 @@ enum class CoincenterCommandType : int8_t { kWithdrawApply, kDustSweeper, + kMarketData, + kReplay, + kReplayMarkets, + kLast }; @@ -39,4 +43,4 @@ std::string_view CoincenterCommandTypeToString(CoincenterCommandType type); CoincenterCommandType CoincenterCommandTypeFromString(std::string_view str); bool IsAnyTrade(CoincenterCommandType type); -} // namespace cct \ No newline at end of file +} // namespace cct diff --git a/src/objects/include/exchange-names.hpp b/src/objects/include/exchange-names.hpp index 339a5e18..b9850579 100644 --- a/src/objects/include/exchange-names.hpp +++ b/src/objects/include/exchange-names.hpp @@ -2,6 +2,8 @@ #include +#include "cct_const.hpp" +#include "cct_fixedcapacityvector.hpp" #include "cct_smallvector.hpp" #include "cct_string.hpp" #include "exchangename.hpp" @@ -11,6 +13,8 @@ namespace cct { using ExchangeNameSpan = std::span; using ExchangeNames = SmallVector; +using PublicExchangeNameVector = FixedCapacityVector; + string ConstructAccumulatedExchangeNames(ExchangeNameSpan exchangeNames); } // namespace cct \ No newline at end of file diff --git a/src/objects/include/exchangeconfig.hpp b/src/objects/include/exchangeconfig.hpp index 17c9375c..008e77cf 100644 --- a/src/objects/include/exchangeconfig.hpp +++ b/src/objects/include/exchangeconfig.hpp @@ -19,7 +19,8 @@ namespace cct { class ExchangeConfig { public: - enum struct FeeType { kMaker, kTaker }; + enum class FeeType : int8_t { kMaker, kTaker }; + enum class MarketDataSerialization : int8_t { kYes, kNo }; struct APIUpdateFrequencies { Duration freq[api::kQueryTypeMax]; @@ -32,7 +33,8 @@ class ExchangeConfig { std::string_view acceptEncoding, int dustSweeperMaxNbTrades, log::level::level_enum requestsCallLogLevel, log::level::level_enum requestsAnswerLogLevel, bool multiTradeAllowedByDefault, bool validateDepositAddressesInFile, bool placeSimulateRealOrder, - bool validateApiKey, TradeConfig tradeConfig, HttpConfig httpConfig); + bool validateApiKey, TradeConfig tradeConfig, HttpConfig httpConfig, + MarketDataSerialization marketDataSerialization); /// Get a reference to the list of statically excluded currency codes to consider for the exchange, /// In both trading and withdrawal. @@ -107,6 +109,8 @@ class ExchangeConfig { PermanentCurlOptions::Builder curlOptionsBuilderBase(Api api) const; + bool withMarketDataSerialization() const { return _withMarketSerialization; } + private: CurrencyCodeSet _excludedCurrenciesAll; // Currencies will be completely ignored by the exchange CurrencyCodeSet _excludedCurrenciesWithdrawal; // Currencies unavailable for withdrawals @@ -128,5 +132,6 @@ class ExchangeConfig { bool _validateDepositAddressesInFile; bool _placeSimulateRealOrder; bool _validateApiKey; + bool _withMarketSerialization; }; } // namespace cct diff --git a/src/objects/include/generalconfig.hpp b/src/objects/include/generalconfig.hpp index 0fd87678..1a30b897 100644 --- a/src/objects/include/generalconfig.hpp +++ b/src/objects/include/generalconfig.hpp @@ -7,6 +7,7 @@ #include "logginginfo.hpp" #include "requestsconfig.hpp" #include "timedef.hpp" +#include "trading-config.hpp" namespace cct { @@ -18,13 +19,15 @@ class GeneralConfig { GeneralConfig() = default; - GeneralConfig(LoggingInfo &&loggingInfo, RequestsConfig &&requestsConfig, Duration fiatConversionQueryRate, - ApiOutputType apiOutputType); + GeneralConfig(LoggingInfo &&loggingInfo, RequestsConfig &&requestsConfig, TradingConfig &&tradingConfig, + Duration fiatConversionQueryRate, ApiOutputType apiOutputType); const LoggingInfo &loggingInfo() const { return _loggingInfo; } const RequestsConfig &requestsConfig() const { return _requestsConfig; } + const TradingConfig &tradingConfig() const { return _tradingConfig; } + ApiOutputType apiOutputType() const { return _apiOutputType; } Duration fiatConversionQueryRate() const { return _fiatConversionQueryRate; } @@ -32,6 +35,7 @@ class GeneralConfig { private: LoggingInfo _loggingInfo{LoggingInfo::WithLoggersCreation::kYes}; RequestsConfig _requestsConfig; + TradingConfig _tradingConfig; Duration _fiatConversionQueryRate = std::chrono::hours(8); ApiOutputType _apiOutputType = ApiOutputType::kFormattedTable; }; diff --git a/src/objects/include/generalconfigdefault.hpp b/src/objects/include/generalconfigdefault.hpp index b4e6b7ef..460e0ec5 100644 --- a/src/objects/include/generalconfigdefault.hpp +++ b/src/objects/include/generalconfigdefault.hpp @@ -35,6 +35,17 @@ struct GeneralConfigDefault { "concurrency": { "nbMaxParallelRequests": 1 } + }, + "trading": { + "automation": { + "deserialization": { + "loadChunkDuration": "1w" + }, + "startingContext": { + "startBaseAmountEquivalent": "1000 EUR", + "startQuoteAmountEquivalent": "1000 EUR" + } + } } } )"_json; diff --git a/src/objects/include/marketorderbook.hpp b/src/objects/include/marketorderbook.hpp index f72a790b..7d74f5aa 100644 --- a/src/objects/include/marketorderbook.hpp +++ b/src/objects/include/marketorderbook.hpp @@ -50,6 +50,8 @@ class MarketOrderBook { int size() const { return _orders.size(); } /// Check if data stored in this MarketOrderBook is valid. + /// This is especially useful for optional check of data after deserialization, + /// as for the standard case the market order book should be valid by design. bool isValid() const; bool isArtificiallyExtended() const { return _isArtificiallyExtended; } @@ -192,6 +194,12 @@ class MarketOrderBook { /// 0.35 20 /// 0.34 23 + // To allow faster MarketOrderBook constructs + friend class MarketOrderBookConverter; + + MarketOrderBook(TimePoint timeStamp, Market market, AmountPriceVector&& orders, int32_t highestBidPricePos, + int32_t lowestAskPricePos, VolAndPriNbDecimals volAndPriNbDecimals); + MonetaryAmount amountAt(int pos) const { return MonetaryAmount(_orders[pos].amount, _market.base(), _volAndPriNbDecimals.volNbDecimals); } diff --git a/src/objects/include/time-window.hpp b/src/objects/include/time-window.hpp new file mode 100644 index 00000000..4d641d90 --- /dev/null +++ b/src/objects/include/time-window.hpp @@ -0,0 +1,70 @@ +#pragma once + +#include +#include + +#include "cct_format.hpp" +#include "cct_invalid_argument_exception.hpp" +#include "timedef.hpp" + +namespace cct { + +/// Simple utility class representing a time window with a beginning time, and an end time. +/// The beginning time includes the corresponding time point, but the end time excludes it. +class TimeWindow { + public: + /// Create a zero duration time window starting from the zero-initialized time point. + TimeWindow() noexcept = default; + + /// Create a time window spanning from 'from' (included) to 'to' (excluded) time points. + TimeWindow(TimePoint from, TimePoint to) : _from(from), _to(to) { + if (_to < _from) { + throw invalid_argument("Invalid time window - 'from' should not be larger than 'to'"); + } + } + + /// Create a time window starting at 'from' with 'dur' duration. + TimeWindow(TimePoint from, Duration dur) : TimeWindow(from, from + dur) {} + + TimePoint from() const { return _from; } + + TimePoint to() const { return _to; } + + Duration duration() const { return _to - _from; } + + bool contains(TimePoint tp) const { return _from <= tp && tp < _to; } + + bool contains(int64_t unixTimestampInMs) const { return contains(TimePoint(milliseconds{unixTimestampInMs})); } + + bool contains(TimeWindow rhs) const { return _from <= rhs._from && rhs._to <= _to; } + + bool overlaps(TimeWindow rhs) const { return _from < rhs._to && rhs._from < _to; } + + string str() const; + + bool operator==(const TimeWindow&) const noexcept = default; + + private: + TimePoint _from; + TimePoint _to; +}; +} // namespace cct + +#ifndef CCT_DISABLE_SPDLOG +template <> +struct fmt::formatter { + constexpr auto parse(format_parse_context& ctx) -> decltype(ctx.begin()) { + const auto it = ctx.begin(); + const auto end = ctx.end(); + if (it != end && *it != '}') { + throw format_error("invalid format"); + } + return it; + } + + template + auto format(const cct::TimeWindow& timeWindow, FormatContext& ctx) const -> decltype(ctx.out()) { + return fmt::format_to(ctx.out(), "{}", timeWindow.str()); + } +}; +#endif diff --git a/src/objects/include/trading-config.hpp b/src/objects/include/trading-config.hpp new file mode 100644 index 00000000..b265e9dc --- /dev/null +++ b/src/objects/include/trading-config.hpp @@ -0,0 +1,19 @@ +#pragma once + +#include + +#include "automation-config.hpp" + +namespace cct { +class TradingConfig { + public: + TradingConfig() noexcept = default; + + TradingConfig(AutomationConfig automationConfig) : _automationConfig(std::move(automationConfig)) {} + + const AutomationConfig &automationConfig() const { return _automationConfig; } + + private: + AutomationConfig _automationConfig; +}; +} // namespace cct \ No newline at end of file diff --git a/src/objects/src/coincentercommandtype.cpp b/src/objects/src/coincentercommandtype.cpp index 8ec46ee1..bbb481ef 100644 --- a/src/objects/src/coincentercommandtype.cpp +++ b/src/objects/src/coincentercommandtype.cpp @@ -16,8 +16,7 @@ constexpr std::string_view kCommandTypeNames[] = { "Balance", "DepositInfo", "OrdersClosed", "OrdersOpened", "OrdersCancel", "RecentDeposits", "RecentWithdraws", "Trade", "Buy", "Sell", - "Withdraw", "DustSweeper", -}; + "Withdraw", "DustSweeper", "MarketData", "Replay", "ReplayMarkets"}; static_assert(std::size(kCommandTypeNames) == static_cast(CoincenterCommandType::kLast)); } // namespace @@ -51,4 +50,4 @@ bool IsAnyTrade(CoincenterCommandType type) { return false; } } -} // namespace cct \ No newline at end of file +} // namespace cct diff --git a/src/objects/src/exchangeconfig.cpp b/src/objects/src/exchangeconfig.cpp index 61c1b4ab..ce9e8406 100644 --- a/src/objects/src/exchangeconfig.cpp +++ b/src/objects/src/exchangeconfig.cpp @@ -67,7 +67,8 @@ ExchangeConfig::ExchangeConfig( const APIUpdateFrequencies &apiUpdateFrequencies, Duration publicAPIRate, Duration privateAPIRate, std::string_view acceptEncoding, int dustSweeperMaxNbTrades, log::level::level_enum requestsCallLogLevel, log::level::level_enum requestsAnswerLogLevel, bool multiTradeAllowedByDefault, bool validateDepositAddressesInFile, - bool placeSimulateRealOrder, bool validateApiKey, TradeConfig tradeConfig, HttpConfig httpConfig) + bool placeSimulateRealOrder, bool validateApiKey, TradeConfig tradeConfig, HttpConfig httpConfig, + MarketDataSerialization marketDataSerialization) : _excludedCurrenciesAll(std::move(excludedAllCurrencies)), _excludedCurrenciesWithdrawal(std::move(excludedCurrenciesWithdraw)), _preferredPaymentCurrencies(std::move(preferredPaymentCurrencies)), @@ -86,7 +87,8 @@ ExchangeConfig::ExchangeConfig( _multiTradeAllowedByDefault(multiTradeAllowedByDefault), _validateDepositAddressesInFile(validateDepositAddressesInFile), _placeSimulateRealOrder(placeSimulateRealOrder), - _validateApiKey(validateApiKey) { + _validateApiKey(validateApiKey), + _withMarketSerialization(marketDataSerialization == MarketDataSerialization::kYes) { if (dustSweeperMaxNbTrades > std::numeric_limits::max() || dustSweeperMaxNbTrades < 0) { throw exception("Invalid number of dust sweeper max trades '{}', should be in [0, {}]", dustSweeperMaxNbTrades, std::numeric_limits::max()); @@ -113,6 +115,7 @@ ExchangeConfig::ExchangeConfig( _validateDepositAddressesInFile ? kDepositAddressesFileName : ""); log::trace(" - Order placing in simulation : {}", _placeSimulateRealOrder ? "real, unmatchable" : "none"); log::trace(" - Validate API Key : {}", _validateApiKey ? "yes" : "no"); + log::trace(" - Market data serialization : {}", _withMarketSerialization ? "yes" : "no"); } if (_preferredPaymentCurrencies.empty()) { log::warn("{} list of preferred currencies is empty, buy and sell commands cannot perform trades", exchangeNameStr); diff --git a/src/objects/src/exchangeconfigdefault.hpp b/src/objects/src/exchangeconfigdefault.hpp index 3199cbd1..93e48258 100644 --- a/src/objects/src/exchangeconfigdefault.hpp +++ b/src/objects/src/exchangeconfigdefault.hpp @@ -59,6 +59,7 @@ struct ExchangeConfigDefault { "requestsCall": "info", "requestsAnswer": "trace" }, + "marketDataSerialization": true, "multiTradeAllowedByDefault": false, "placeSimulateRealOrder": false, "trade": { @@ -186,6 +187,7 @@ struct ExchangeConfigDefault { "requestsCall": "info", "requestsAnswer": "trace" }, + "marketDataSerialization": false, "multiTradeAllowedByDefault": true, "privateAPIRate": "1055ms", "publicAPIRate": "1236ms", diff --git a/src/objects/src/exchangeconfigmap.cpp b/src/objects/src/exchangeconfigmap.cpp index d9b5a472..16f9e6f6 100644 --- a/src/objects/src/exchangeconfigmap.cpp +++ b/src/objects/src/exchangeconfigmap.cpp @@ -58,6 +58,10 @@ ExchangeConfigMap ComputeExchangeConfigMap(std::string_view fileName, const json withdrawTopLevelOption.getBool(exchangeName, "validateDepositAddressesInFile"); const bool placeSimulatedRealOrder = queryTopLevelOption.getBool(exchangeName, "placeSimulateRealOrder"); const bool validateApiKey = queryTopLevelOption.getBool(exchangeName, "validateApiKey"); + const ExchangeConfig::MarketDataSerialization marketDataSerialization = + queryTopLevelOption.getBool(exchangeName, "marketDataSerialization") + ? ExchangeConfig::MarketDataSerialization::kYes + : ExchangeConfig::MarketDataSerialization::kNo; MonetaryAmountByCurrencySet dustAmountsThresholds( queryTopLevelOption.getMonetaryAmountsArray(exchangeName, "dustAmountsThreshold")); @@ -90,7 +94,7 @@ ExchangeConfigMap ComputeExchangeConfigMap(std::string_view fileName, const json std::move(dustAmountsThresholds), std::move(apiUpdateFrequencies), publicAPIRate, privateAPIRate, acceptEncoding, dustSweeperMaxNbTrades, requestsCallLogLevel, requestsAnswerLogLevel, multiTradeAllowedByDefault, validateDepositAddressesInFile, placeSimulatedRealOrder, - validateApiKey, std::move(tradeConfig), std::move(httpConfig))); + validateApiKey, std::move(tradeConfig), std::move(httpConfig), marketDataSerialization)); } // namespace cct // Print json unused values @@ -120,4 +124,4 @@ ExchangeConfigMap ComputeExchangeConfigMap(std::string_view fileName, const json return map; } -} // namespace cct \ No newline at end of file +} // namespace cct diff --git a/src/objects/src/generalconfig.cpp b/src/objects/src/generalconfig.cpp index c08c3270..96494ec1 100644 --- a/src/objects/src/generalconfig.cpp +++ b/src/objects/src/generalconfig.cpp @@ -11,13 +11,15 @@ #include "logginginfo.hpp" #include "requestsconfig.hpp" #include "timedef.hpp" +#include "trading-config.hpp" namespace cct { -GeneralConfig::GeneralConfig(LoggingInfo &&loggingInfo, RequestsConfig &&requestsConfig, +GeneralConfig::GeneralConfig(LoggingInfo &&loggingInfo, RequestsConfig &&requestsConfig, TradingConfig &&tradingConfig, Duration fiatConversionQueryRate, ApiOutputType apiOutputType) : _loggingInfo(std::move(loggingInfo)), _requestsConfig(std::move(requestsConfig)), + _tradingConfig(std::move(tradingConfig)), _fiatConversionQueryRate(fiatConversionQueryRate), _apiOutputType(apiOutputType) {} diff --git a/src/objects/src/marketorderbook.cpp b/src/objects/src/marketorderbook.cpp index 89791843..b4fd14fa 100644 --- a/src/objects/src/marketorderbook.cpp +++ b/src/objects/src/marketorderbook.cpp @@ -236,6 +236,16 @@ MarketOrderBook::MarketOrderBook(TimePoint timeStamp, MonetaryAmount askPrice, M } } +MarketOrderBook::MarketOrderBook(TimePoint timeStamp, Market market, AmountPriceVector&& orders, + int32_t highestBidPricePos, int32_t lowestAskPricePos, + VolAndPriNbDecimals volAndPriNbDecimals) + : _time(timeStamp), + _market(market), + _orders(std::move(orders)), + _highestBidPricePos(highestBidPricePos), + _lowestAskPricePos(lowestAskPricePos), + _volAndPriNbDecimals(volAndPriNbDecimals) {} + bool MarketOrderBook::isValid() const { if (_orders.size() < 2U) { log::error("Market order book is invalid as size is {}", _orders.size()); diff --git a/src/objects/src/publictrade.cpp b/src/objects/src/publictrade.cpp index dd023f84..c1cc9ba0 100644 --- a/src/objects/src/publictrade.cpp +++ b/src/objects/src/publictrade.cpp @@ -34,4 +34,4 @@ bool PublicTrade::isValid() const { return true; } -} // namespace cct \ No newline at end of file +} // namespace cct diff --git a/src/objects/src/time-window.cpp b/src/objects/src/time-window.cpp new file mode 100644 index 00000000..088281f6 --- /dev/null +++ b/src/objects/src/time-window.cpp @@ -0,0 +1,15 @@ +#include "time-window.hpp" + +#include "timestring.hpp" + +namespace cct { +string TimeWindow::str() const { + string ret; + ret.push_back('['); + ret.append(ToString(from(), kTimeYearToSecondSpaceSeparatedFormat)); + ret.append(" -> "); + ret.append(ToString(to(), kTimeYearToSecondSpaceSeparatedFormat)); + ret.push_back(')'); + return ret; +} +} // namespace cct \ No newline at end of file diff --git a/src/objects/test/time-window_test.cpp b/src/objects/test/time-window_test.cpp new file mode 100644 index 00000000..42f6b04a --- /dev/null +++ b/src/objects/test/time-window_test.cpp @@ -0,0 +1,140 @@ +#include "time-window.hpp" + +#include + +#include + +#include "cct_invalid_argument_exception.hpp" +#include "timedef.hpp" + +namespace cct { +class TimeWindowTest : public ::testing::Test { + protected: + TimePoint tp1{milliseconds{std::numeric_limits::max() / 10000000}}; + TimePoint tp2{milliseconds{std::numeric_limits::max() / 9900000}}; + TimePoint tp3{milliseconds{std::numeric_limits::max() / 9800000}}; + TimePoint tp4{milliseconds{std::numeric_limits::max() / 9500000}}; + TimePoint tp5{milliseconds{std::numeric_limits::max() / 9000000}}; + + Duration dur1{seconds{100}}; + Duration dur2{seconds{1000}}; + Duration dur3{seconds{10000}}; +}; + +TEST_F(TimeWindowTest, DefaultConstructor) { + TimeWindow tw; + + EXPECT_EQ(tw.from(), TimePoint{}); + EXPECT_EQ(tw.to(), TimePoint{}); + EXPECT_EQ(tw.duration(), milliseconds{}); + EXPECT_FALSE(tw.contains(TimePoint{})); + EXPECT_FALSE(tw.contains(0)); + EXPECT_TRUE(tw.contains(tw)); +} + +TEST_F(TimeWindowTest, InvalidTimeWindowFromTime) { EXPECT_THROW(TimeWindow(tp2, tp1), invalid_argument); } +TEST_F(TimeWindowTest, InvalidTimeWindowFromDuration) { EXPECT_THROW(TimeWindow(tp1, tp1 - tp2), invalid_argument); } + +TEST_F(TimeWindowTest, DurationConstructor) { + TimeWindow tw(tp1, tp2 - tp1); + + EXPECT_EQ(tw, TimeWindow(tp1, tp2)); +} + +TEST_F(TimeWindowTest, Duration) { + TimeWindow tw(tp1, tp2); + + EXPECT_EQ(tw.duration(), tp2 - tp1); +} + +TEST_F(TimeWindowTest, ContainsTimePoint) { + TimeWindow tw1(tp1, tp2); + + EXPECT_TRUE(tw1.contains(tp1)); + EXPECT_TRUE(tw1.contains(tp1 + dur1)); + EXPECT_FALSE(tw1.contains(tp2)); + EXPECT_FALSE(tw1.contains(tp3)); +} + +TEST_F(TimeWindowTest, ContainsTimeWindow) { + // [ ] + // [ ] + TimeWindow tw1(tp1, tp4); + TimeWindow tw2(tp2, tp3); + + EXPECT_TRUE(tw1.contains(tw1)); + EXPECT_TRUE(tw1.overlaps(tw1)); + + EXPECT_TRUE(tw1.overlaps(tw2)); + EXPECT_TRUE(tw1.contains(tw2)); + + EXPECT_TRUE(tw2.overlaps(tw1)); + EXPECT_FALSE(tw2.contains(tw1)); +} + +TEST_F(TimeWindowTest, OverlapNominal) { + // [ ] + // [ ] + TimeWindow tw1(tp2, tp4); + TimeWindow tw2(tp1, tp3); + + EXPECT_TRUE(tw1.overlaps(tw2)); + EXPECT_FALSE(tw1.contains(tw2)); + + EXPECT_TRUE(tw2.overlaps(tw1)); + EXPECT_FALSE(tw2.contains(tw1)); +} + +TEST_F(TimeWindowTest, OverlapEqualTo) { + // [ ] + // [ ] + TimeWindow tw1(tp1, tp3); + TimeWindow tw2(tp2, tp3); + + EXPECT_TRUE(tw1.overlaps(tw2)); + EXPECT_TRUE(tw1.contains(tw2)); + + EXPECT_TRUE(tw2.overlaps(tw1)); + EXPECT_FALSE(tw2.contains(tw1)); +} + +TEST_F(TimeWindowTest, OverlapEqualFrom) { + // [ ] + // [ ] + TimeWindow tw1(tp1, tp3); + TimeWindow tw2(tp1, tp2); + + EXPECT_TRUE(tw1.overlaps(tw2)); + EXPECT_TRUE(tw1.contains(tw2)); + + EXPECT_TRUE(tw2.overlaps(tw1)); + EXPECT_FALSE(tw2.contains(tw1)); +} + +TEST_F(TimeWindowTest, NoOverlapNominal) { + // [ ] + // [ ] + TimeWindow tw1(tp1, tp2); + TimeWindow tw2(tp3, tp4); + + EXPECT_FALSE(tw1.overlaps(tw2)); + EXPECT_FALSE(tw1.contains(tw2)); + + EXPECT_FALSE(tw2.overlaps(tw1)); + EXPECT_FALSE(tw2.contains(tw1)); +} + +TEST_F(TimeWindowTest, NoOverlapEqual) { + // [ ] + // [ ] + TimeWindow tw1(tp1, tp3); + TimeWindow tw2(tp3, tp4); + + EXPECT_FALSE(tw1.overlaps(tw2)); + EXPECT_FALSE(tw1.contains(tw2)); + + EXPECT_FALSE(tw2.overlaps(tw1)); + EXPECT_FALSE(tw2.contains(tw1)); +} + +} // namespace cct diff --git a/src/serialization/CMakeLists.txt b/src/serialization/CMakeLists.txt new file mode 100644 index 00000000..1724597e --- /dev/null +++ b/src/serialization/CMakeLists.txt @@ -0,0 +1,74 @@ +if(CCT_ENABLE_PROTO) + aux_source_directory(src SERIALIZATION_SRC) + + list(APPEND SERIALIZATION_SRC "${CMAKE_CURRENT_LIST_DIR}/proto/market-order-book.proto") + list(APPEND SERIALIZATION_SRC "${CMAKE_CURRENT_LIST_DIR}/proto/public-trade.proto") +else() + set(SERIALIZATION_SRC "src/dummy-market-data-serializer.cpp" "src/dummy-market-data-deserializer.cpp") +endif() + +add_library(coincenter_serialization STATIC ${SERIALIZATION_SRC}) + +target_include_directories(coincenter_serialization PUBLIC include) +target_link_libraries(coincenter_serialization PUBLIC coincenter_objects) + +if(CCT_ENABLE_PROTO) + set(PROTO_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/generated") + + target_include_directories(coincenter_serialization PUBLIC "$") + + target_link_libraries(coincenter_serialization PUBLIC protobuf::libprotobuf) + + if(MSVC) + # Not sure why it's needed for Windows. + target_link_libraries(libprotobuf PUBLIC ZLIB::ZLIB) + endif() + target_link_libraries(coincenter_serialization PUBLIC ZLIB::ZLIB) + + protobuf_generate( + TARGET coincenter_serialization + IMPORT_DIRS "${CMAKE_CURRENT_LIST_DIR}/proto" + PROTOC_OUT_DIR "${PROTO_BINARY_DIR}" + ) + + add_unit_test( + continuous-iterator_test + test/continuous-iterator_test.cpp + ) + + add_unit_test( + proto-market-order-book-converter_test + test/proto-market-order-book-converter_test.cpp + LIBRARIES + coincenter_serialization + ) + + add_unit_test( + proto-multiple-messages-handler_test + test/proto-multiple-messages-handler_test.cpp + LIBRARIES + coincenter_serialization + ) + + add_unit_test( + proto-public-trade-converter_test + test/proto-public-trade-converter_test.cpp + LIBRARIES + coincenter_serialization + ) + + add_unit_test( + proto-serialization-and-deserialization_test + test/proto-serialization-and-deserialization_test.cpp + LIBRARIES + coincenter_serialization + ) + + add_unit_test( + serialization-tools_test + test/serialization-tools_test.cpp + LIBRARIES + coincenter_serialization + ) + +endif() diff --git a/src/serialization/include/abstract-market-data-deserializer.hpp b/src/serialization/include/abstract-market-data-deserializer.hpp new file mode 100644 index 00000000..fd66d9d1 --- /dev/null +++ b/src/serialization/include/abstract-market-data-deserializer.hpp @@ -0,0 +1,24 @@ +#pragma once + +#include "market-order-book-vector.hpp" +#include "market-timestamp-set.hpp" +#include "market.hpp" +#include "public-trade-vector.hpp" +#include "time-window.hpp" + +namespace cct { + +class AbstractMarketDataDeserializer { + public: + virtual ~AbstractMarketDataDeserializer() = default; + + virtual MarketTimestampSet pullMarketOrderBooksMarkets(TimeWindow timeWindow) = 0; + + virtual MarketTimestampSet pullTradeMarkets(TimeWindow timeWindow) = 0; + + virtual MarketOrderBookVector pullMarketOrderBooks(Market market, TimeWindow timeWindow) = 0; + + virtual PublicTradeVector pullTrades(Market market, TimeWindow timeWindow) = 0; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/include/abstract-market-data-serializer.hpp b/src/serialization/include/abstract-market-data-serializer.hpp new file mode 100644 index 00000000..65cc1b79 --- /dev/null +++ b/src/serialization/include/abstract-market-data-serializer.hpp @@ -0,0 +1,24 @@ +#pragma once + +#include + +#include "market.hpp" +#include "publictrade.hpp" + +namespace cct { + +class MarketOrderBook; + +class AbstractMarketDataSerializer { + public: + virtual ~AbstractMarketDataSerializer() = default; + + /// Push market order book in the MarketDataSerializer. + virtual void push(const MarketOrderBook &marketOrderBook) = 0; + + /// Push public trades in the MarketDataSerializer. + /// They should come from the same market. + virtual void push(Market market, std::span publicTrades) = 0; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/include/continuous-iterator.hpp b/src/serialization/include/continuous-iterator.hpp new file mode 100644 index 00000000..91b71cd1 --- /dev/null +++ b/src/serialization/include/continuous-iterator.hpp @@ -0,0 +1,23 @@ +#pragma once + +namespace cct { + +/// Simple utility class that may iterate in both directions. +class ContinuousIterator { + public: + ContinuousIterator(int from, int to) : _to(to), _curr(from), _incr(to < from ? -1 : 1) {} + + bool hasNext() const { return _curr != _to + _incr; } + + auto next() { + _curr += _incr; + return _curr - _incr; + } + + private: + int _to; + int _curr; + int _incr; +}; + +} // namespace cct diff --git a/src/serialization/include/dummy-market-data-deserializer.hpp b/src/serialization/include/dummy-market-data-deserializer.hpp new file mode 100644 index 00000000..e3b0227c --- /dev/null +++ b/src/serialization/include/dummy-market-data-deserializer.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include + +#include "abstract-market-data-deserializer.hpp" +#include "market-order-book-vector.hpp" +#include "market-timestamp-set.hpp" +#include "market.hpp" +#include "public-trade-vector.hpp" +#include "time-window.hpp" + +namespace cct { + +class DummyMarketDataDeserializer : public AbstractMarketDataDeserializer { + public: + DummyMarketDataDeserializer([[maybe_unused]] std::string_view dataDir, + [[maybe_unused]] std::string_view exchangeName); + + MarketTimestampSet pullMarketOrderBooksMarkets([[maybe_unused]] TimeWindow timeWindow) override; + + MarketTimestampSet pullTradeMarkets([[maybe_unused]] TimeWindow timeWindow) override; + + MarketOrderBookVector pullMarketOrderBooks([[maybe_unused]] Market market, + [[maybe_unused]] TimeWindow timeWindow) override; + + PublicTradeVector pullTrades([[maybe_unused]] Market market, [[maybe_unused]] TimeWindow timeWindow) override; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/include/dummy-market-data-serializer.hpp b/src/serialization/include/dummy-market-data-serializer.hpp new file mode 100644 index 00000000..7665cae3 --- /dev/null +++ b/src/serialization/include/dummy-market-data-serializer.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include +#include + +#include "abstract-market-data-serializer.hpp" +#include "market-timestamp-set.hpp" +#include "market.hpp" +#include "publictrade.hpp" + +namespace cct { + +class MarketOrderBook; + +/// Implementation of a market data serializer that does nothing. +/// Useful if coincenter is not compiled with protobuf support. +class DummyMarketDataSerializer : public AbstractMarketDataSerializer { + public: + DummyMarketDataSerializer(std::string_view dataDir, const MarketTimestampSets &lastWrittenObjectsMarketTimestamp, + std::string_view exchangeName); + + void push(const MarketOrderBook &marketOrderBook) override; + + void push(Market market, std::span publicTrades) override; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/include/market-timestamp-set.hpp b/src/serialization/include/market-timestamp-set.hpp new file mode 100644 index 00000000..afda7143 --- /dev/null +++ b/src/serialization/include/market-timestamp-set.hpp @@ -0,0 +1,15 @@ +#pragma once + +#include "cct_flatset.hpp" +#include "market-timestamp.hpp" + +namespace cct { + +using MarketTimestampSet = FlatSet; + +struct MarketTimestampSets { + MarketTimestampSet orderBooksMarkets; + MarketTimestampSet tradesMarkets; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/include/market-timestamp.hpp b/src/serialization/include/market-timestamp.hpp new file mode 100644 index 00000000..67317c69 --- /dev/null +++ b/src/serialization/include/market-timestamp.hpp @@ -0,0 +1,17 @@ +#pragma once + +#include + +#include "market.hpp" +#include "timedef.hpp" + +namespace cct { + +struct MarketTimestamp { + Market market; + TimePoint timePoint; + + std::strong_ordering operator<=>(const MarketTimestamp &) const noexcept = default; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/include/proto-constants.hpp b/src/serialization/include/proto-constants.hpp new file mode 100644 index 00000000..28a48461 --- /dev/null +++ b/src/serialization/include/proto-constants.hpp @@ -0,0 +1,15 @@ +#pragma once + +#include +#include + +namespace cct { + +enum class ProtobufObject : int8_t { kMarketOrderBook, kTrade }; + +static constexpr std::string_view kBinProtobufExtension = ".binpb"; + +static constexpr std::string_view kSubPathMarketOrderBooks = "order-books"; +static constexpr std::string_view kSubPathTrades = "trades"; + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/include/proto-deserializer.hpp b/src/serialization/include/proto-deserializer.hpp new file mode 100644 index 00000000..0440145e --- /dev/null +++ b/src/serialization/include/proto-deserializer.hpp @@ -0,0 +1,183 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cct_log.hpp" +#include "cct_vector.hpp" +#include "continuous-iterator.hpp" +#include "market-timestamp-set.hpp" +#include "market-timestamp.hpp" +#include "market.hpp" +#include "proto-multiple-messages-handler.hpp" +#include "serialization-tools.hpp" +#include "stringhelpers.hpp" +#include "time-window.hpp" +#include "timedef.hpp" + +namespace cct { + +template +class ProtobufObjectsDeserializer { + public: + using CoincenterObjectType = std::invoke_result_t; + + explicit ProtobufObjectsDeserializer(std::filesystem::path exchangeSerializedDataPath) noexcept + : _exchangeSerializedDataPath(std::move(exchangeSerializedDataPath)) {} + + /// Load all markets found on disk which has some data in the given time window + MarketTimestampSet listMarkets(TimeWindow timeWindow) { + vector marketTimestamps; + + std::error_code ec; + if (std::filesystem::is_directory(_exchangeSerializedDataPath, ec)) { + for (const auto& marketDirectory : std::filesystem::directory_iterator(_exchangeSerializedDataPath)) { + auto ts = loadMarket(marketDirectory, timeWindow, ActionType::kCheckPresence).second; + if (ts != TimePoint{}) { + const auto& marketPath = marketDirectory.path(); + + marketTimestamps.emplace_back(Market{marketPath.filename().string()}, ts); + } + } + } else if (std::filesystem::exists(_exchangeSerializedDataPath)) { + log::error("{} is not a valid directory: {}", _exchangeSerializedDataPath.string(), ec.message()); + } + + return MarketTimestampSet(std::move(marketTimestamps)); + } + + /// Load all data found on disk for given market for the time window + vector loadMarket(Market market, TimeWindow timeWindow) { + const std::filesystem::path marketPath = _exchangeSerializedDataPath / std::string_view{market.str()}; + + return loadMarket(std::filesystem::directory_entry(marketPath), timeWindow, ActionType::kLoad).first; + } + + private: + static bool ValidateTimestamp(const ProtobufObjType& msg, TimeWindow timeWindow) { + if (!msg.has_unixtimestampinms()) { + log::error("Invalid data loaded for protobuf object, no unix timestamp set"); + return false; + } + return timeWindow.contains(msg.unixtimestampinms()); + } + + enum class ActionType : int8_t { kLoad, kCheckPresence }; + + static ContinuousIterator CreateIt(int from, int to, ActionType actionType) { + if (actionType == ActionType::kCheckPresence) { + // In check presence mode, we are only interested in the latest timestamp, so we explore towards the past + std::swap(from, to); + } + return {from, to}; + } + + /// Load all data found on disk for given market for the time window + auto loadMarket(const std::filesystem::directory_entry& marketDirectory, TimeWindow timeWindow, + ActionType actionType) { + std::pair, TimePoint> ret; + if (!marketDirectory.is_directory()) { + return ret; + } + const auto fromDays = std::chrono::floor(timeWindow.from()); + const std::chrono::year_month_day fromYmd{fromDays}; + const std::chrono::hh_mm_ss fromTime{std::chrono::floor(timeWindow.from() - fromDays)}; + + const auto toDays = std::chrono::floor(timeWindow.to()); + const std::chrono::year_month_day toYmd{toDays}; + const std::chrono::hh_mm_ss toTime{std::chrono::floor(timeWindow.to() - toDays)}; + + const auto& marketPath = marketDirectory.path(); + const auto marketFilename = marketPath.filename(); + const Market market(marketFilename.string()); + + ProtoToCoincenterObjectsFunc converter(market); + + const int fromYear = static_cast(fromYmd.year()); + const int toYear = static_cast(toYmd.year()); + + for (ContinuousIterator yearIt = CreateIt(fromYear, toYear, actionType); yearIt.hasNext();) { + const auto year = yearIt.next(); + const auto yearPath = marketPath / std::string_view(ToCharVector(year)); + if (!std::filesystem::is_directory(yearPath)) { + continue; + } + const bool isYearFromExtremity = year == fromYear; + const bool isYearToExtremity = year == toYear; + const auto fromMonth = isYearFromExtremity ? static_cast(static_cast(fromYmd.month())) : 1; + const auto toMonth = isYearToExtremity ? static_cast(static_cast(toYmd.month())) : 12; + + for (ContinuousIterator monthIt = CreateIt(fromMonth, toMonth, actionType); monthIt.hasNext();) { + const auto month = monthIt.next(); + const auto monthPath = yearPath / MonthStr(month); + if (!std::filesystem::is_directory(monthPath)) { + continue; + } + const bool isMonthFromExtremity = isYearFromExtremity && month == fromMonth; + const bool isMonthToExtremity = isYearToExtremity && month == toMonth; + const auto fromDay = isMonthFromExtremity ? static_cast(static_cast(fromYmd.day())) : 1; + const auto toDay = isMonthToExtremity ? static_cast(static_cast(toYmd.day())) : 31; + + for (ContinuousIterator dayOfMonthIt = CreateIt(fromDay, toDay, actionType); dayOfMonthIt.hasNext();) { + const auto dayOfMonth = dayOfMonthIt.next(); + const auto dayPath = monthPath / DayOfMonthStr(dayOfMonth); + if (!std::filesystem::is_directory(dayPath)) { + continue; + } + + const bool isDayFromExtremity = isMonthFromExtremity && dayOfMonth == fromDay; + const bool isDayToExtremity = isMonthToExtremity && dayOfMonth == toDay; + const auto fromHour = isDayFromExtremity ? static_cast(fromTime.hours().count()) : 0; + const auto toHour = isDayToExtremity ? static_cast(toTime.hours().count()) : 23; + + for (ContinuousIterator hourOfDayIt = CreateIt(fromHour, toHour, actionType); hourOfDayIt.hasNext();) { + const auto hourOfDay = hourOfDayIt.next(); + const auto hourPath = dayPath / ComputeProtoFileName(hourOfDay); + if (!std::filesystem::exists(hourPath)) { + continue; + } + + decltype(std::declval().unixtimestampinms()) lastTs = 0; + + std::ifstream ifs(hourPath, std::ios::in | std::ios::binary); + for (ProtobufMessageCompressedReaderIterator protobufMessageReaderIt(ifs); + protobufMessageReaderIt.hasNext();) { + auto msg = protobufMessageReaderIt.next(); + if (!ValidateTimestamp(msg, timeWindow)) { + continue; + } + + // In Check presence mode, we read all the file to retrieve the latest timestamp. + // There's no other way to do it. + lastTs = msg.unixtimestampinms(); + + if (actionType == ActionType::kLoad) { + ret.first.push_back(converter(std::move(msg))); + } + } + + if (lastTs != 0) { + if (ret.second == TimePoint{}) { + ret.second = TimePoint{milliseconds{static_cast(lastTs)}}; + } + if (actionType == ActionType::kCheckPresence) { + return ret; + } + } + } + } + } + } + return ret; + } + + std::filesystem::path _exchangeSerializedDataPath; +}; + +} // namespace cct diff --git a/src/serialization/include/proto-market-data-deserializer.hpp b/src/serialization/include/proto-market-data-deserializer.hpp new file mode 100644 index 00000000..2e653198 --- /dev/null +++ b/src/serialization/include/proto-market-data-deserializer.hpp @@ -0,0 +1,36 @@ +#pragma once + +#include + +#include "abstract-market-data-deserializer.hpp" +#include "market-order-book-vector.hpp" +#include "market-order-book.pb.h" +#include "market-timestamp-set.hpp" +#include "market.hpp" +#include "proto-deserializer.hpp" +#include "proto-market-order-book-converter.hpp" +#include "proto-public-trade-converter.hpp" +#include "public-trade-vector.hpp" +#include "public-trade.pb.h" +#include "time-window.hpp" + +namespace cct { + +class ProtoMarketDataDeserializer : public AbstractMarketDataDeserializer { + public: + ProtoMarketDataDeserializer(std::string_view dataDir, std::string_view exchangeName); + + MarketTimestampSet pullMarketOrderBooksMarkets(TimeWindow timeWindow) override; + + MarketTimestampSet pullTradeMarkets(TimeWindow timeWindow) override; + + MarketOrderBookVector pullMarketOrderBooks(Market market, TimeWindow timeWindow) override; + + PublicTradeVector pullTrades(Market market, TimeWindow timeWindow) override; + + private: + ProtobufObjectsDeserializer<::proto::MarketOrderBook, MarketOrderBookConverter> _marketOrderBookDeserializer; + ProtobufObjectsDeserializer<::proto::PublicTrade, PublicTradeConverter> _publicTradeDeserializer; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/include/proto-market-data-serializer.hpp b/src/serialization/include/proto-market-data-serializer.hpp new file mode 100644 index 00000000..5305709c --- /dev/null +++ b/src/serialization/include/proto-market-data-serializer.hpp @@ -0,0 +1,35 @@ +#pragma once + +#include +#include + +#include "abstract-market-data-serializer.hpp" +#include "market-order-book.pb.h" +#include "market-timestamp-set.hpp" +#include "market.hpp" +#include "proto-public-trade-compare.hpp" +#include "proto-serializer.hpp" +#include "public-trade.pb.h" +#include "publictrade.hpp" + +namespace cct { + +class MarketOrderBook; + +/// This class is responsible of managing the periodic writes to disk of timed market data, for a given exchange. +/// This class is not thread safe +class ProtoMarketDataSerializer : public AbstractMarketDataSerializer { + public: + ProtoMarketDataSerializer(std::string_view dataDir, const MarketTimestampSets &lastWrittenObjectsMarketTimestamp, + std::string_view exchangeName); + + void push(const MarketOrderBook &marketOrderBook) override; + + void push(Market market, std::span publicTrades) override; + + private: + ProtobufObjectsSerializer<::proto::MarketOrderBook> _marketOrderBookSerializer; + ProtobufObjectsSerializer<::proto::PublicTrade, ProtoPublicTradeComp, ProtoPublicTradeEqual> _tradesSerializer; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/include/proto-market-order-book-converter.hpp b/src/serialization/include/proto-market-order-book-converter.hpp new file mode 100644 index 00000000..6cff3bb8 --- /dev/null +++ b/src/serialization/include/proto-market-order-book-converter.hpp @@ -0,0 +1,21 @@ +#pragma once + +#include "market-order-book.pb.h" +#include "market.hpp" +#include "marketorderbook.hpp" + +namespace cct { + +::proto::MarketOrderBook ConvertMarketOrderBookToProto(const MarketOrderBook &marketOrderBook); + +class MarketOrderBookConverter { + public: + explicit MarketOrderBookConverter(Market market) : _market(market) {} + + MarketOrderBook operator()(const ::proto::MarketOrderBook &marketOrderBookTimedData); + + private: + Market _market; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/include/proto-multiple-messages-handler.hpp b/src/serialization/include/proto-multiple-messages-handler.hpp new file mode 100644 index 00000000..19bb1a0e --- /dev/null +++ b/src/serialization/include/proto-multiple-messages-handler.hpp @@ -0,0 +1,199 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cct_exception.hpp" +#include "cct_log.hpp" + +namespace cct { + +/// Utility class that allows compressed data to be written to rolling files. +/// At default constructor, no file is opened. Client is expected to call 'open' with a new std::ostream prior to any +/// write. Data is written with the following scheme: +/// - First a VarInt64 is written with the size (in bytes) of the object to be serialized. +/// - Then the object itself is serialized. +template +class ProtobufMessagesCompressedWriter { + private: + static_assert(std::derived_from, + "OStreamType for ProtobufMessagesCompressedWriter should derive from std::ostream"); + + using OstreamOutputStream = ::google::protobuf::io::OstreamOutputStream; + using GzipOutputStream = ::google::protobuf::io::GzipOutputStream; + using CodedOutputStream = ::google::protobuf::io::CodedOutputStream; + + public: + /// Initializes a new ProtobufMessagesCompressedWriter without any opened stream. + /// 'open' method should be called before any write. + ProtobufMessagesCompressedWriter() noexcept(std::is_nothrow_default_constructible_v) = default; + + /// No copy / move operations allowed as already deleted by underlying OstreamOutputStream & CodedOutputStream + ProtobufMessagesCompressedWriter(const ProtobufMessagesCompressedWriter&) = delete; + ProtobufMessagesCompressedWriter(ProtobufMessagesCompressedWriter&&) = delete; + + ProtobufMessagesCompressedWriter& operator=(const ProtobufMessagesCompressedWriter&) = delete; + ProtobufMessagesCompressedWriter& operator=(ProtobufMessagesCompressedWriter&&) = delete; + + ~ProtobufMessagesCompressedWriter() { close(); } + + void open(OStreamType&& newOs) { + close(); + + _os = std::move(newOs); + + OstreamOutputStream* pOos; + try { + pOos = std::construct_at(getOstreamOutputStreamPtr(), std::addressof(_os)); + } catch (const std::exception& ex) { + _os = OStreamType(); + throw ex; + } + + GzipOutputStream* pGzos; + try { + pGzos = std::construct_at(getGzipOutputStreamPtr(), pOos); + } catch (const std::exception& ex) { + std::destroy_at(pOos); + _os = OStreamType(); + throw ex; + } + + try { + std::construct_at(getCodedOutputStreamPtr(), pGzos); + } catch (const std::exception& ex) { + std::destroy_at(pGzos); + std::destroy_at(pOos); + _os = OStreamType(); + throw ex; + } + + _isFileOpened = true; + } + + template + void write(const MsgT& msg) { + if (!_isFileOpened) { + throw exception("ProtobufMessagesWriter::open should have been called first"); + } + + auto* cos = getCodedOutputStreamPtr(); + + cos->WriteVarint32(static_cast(msg.ByteSizeLong())); + + if (!msg.SerializeToCodedStream(cos)) { + log::error("Failed to serialize to coded stream"); + } + } + + OStreamType flush() noexcept(std::is_nothrow_swappable_v) { + close(); + + OStreamType ret; + ret.swap(_os); + return ret; + } + + private: + void close() noexcept { + if (_isFileOpened) { + // reverse destroy streams to flush latest data. Recreate the streams after creation of new ofstream + std::destroy_at(getCodedOutputStreamPtr()); + std::destroy_at(getGzipOutputStreamPtr()); + std::destroy_at(getOstreamOutputStreamPtr()); + _isFileOpened = false; + } + } + + OstreamOutputStream* getOstreamOutputStreamPtr() { return reinterpret_cast(_oos); } + GzipOutputStream* getGzipOutputStreamPtr() { return reinterpret_cast(_gzos); } + CodedOutputStream* getCodedOutputStreamPtr() { return reinterpret_cast(_cos); } + + OStreamType _os; + alignas(OstreamOutputStream) std::byte _oos[sizeof(OstreamOutputStream)]; + alignas(GzipOutputStream) std::byte _gzos[sizeof(GzipOutputStream)]; + alignas(CodedOutputStream) std::byte _cos[sizeof(CodedOutputStream)]; + bool _isFileOpened = false; +}; + +class ProtobufMessageReaderBase { + public: + explicit ProtobufMessageReaderBase(google::protobuf::io::ZeroCopyInputStream* is) : _cis(is) {} + + /// Tells whether this reader has at least one more message to be read. + bool hasNext() { return _cis.ReadVarintSizeAsInt(&_nextSize); } + + /// Read next message and returns it. + /// 'hasNext' should have been called before, and returned true. + template + MsgT next() { + MsgT msg; + auto msgLimit = _cis.PushLimit(_nextSize); + if (!msg.ParseFromCodedStream(&_cis)) { + log::error("Error reading single protobuf message of size {}", _nextSize); + } + _cis.PopLimit(msgLimit); + return msg; + } + + private: + ::google::protobuf::io::CodedInputStream _cis; + int _nextSize{}; +}; + +/// Uncompressed messages reader iterator, provided as an example. Unused in production code, but can be useful if some +/// data has been written uncompressed. +class ProtobufMessageReaderIterator { + public: + explicit ProtobufMessageReaderIterator(std::istream& is) : _iis(&is), _protobufMessageReaderBase(&_iis) {} + + /// Tells whether this reader has at least one more message to be read. + bool hasNext() { return _protobufMessageReaderBase.hasNext(); } + + /// Read next message and returns it. + /// 'hasNext' should have been called before, and returned true. + template + MsgT next() { + return _protobufMessageReaderBase.next(); + } + + private: + ::google::protobuf::io::IstreamInputStream _iis; + ProtobufMessageReaderBase _protobufMessageReaderBase; +}; + +/// The compressed reader iterator that should be used in case files have been written with a +/// ProtobufMessagesCompressedWriter. +class ProtobufMessageCompressedReaderIterator { + public: + explicit ProtobufMessageCompressedReaderIterator(std::istream& is) + : _iis(&is), _gzipIs(&_iis), _protobufMessageReaderBase(&_gzipIs) {} + + /// Tells whether this reader has at least one more message to be read. + bool hasNext() { return _protobufMessageReaderBase.hasNext(); } + + /// Read next message and returns it. + /// 'hasNext' should have been called before, and returned true. + template + MsgT next() { + return _protobufMessageReaderBase.next(); + } + + private: + ::google::protobuf::io::IstreamInputStream _iis; + ::google::protobuf::io::GzipInputStream _gzipIs; + ProtobufMessageReaderBase _protobufMessageReaderBase; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/include/proto-public-trade-compare.hpp b/src/serialization/include/proto-public-trade-compare.hpp new file mode 100644 index 00000000..812a5a1f --- /dev/null +++ b/src/serialization/include/proto-public-trade-compare.hpp @@ -0,0 +1,17 @@ +#pragma once + +namespace proto { +class PublicTrade; +} + +namespace cct { + +struct ProtoPublicTradeComp { + bool operator()(const ::proto::PublicTrade &lhs, const ::proto::PublicTrade &rhs) const; +}; + +struct ProtoPublicTradeEqual { + bool operator()(const ::proto::PublicTrade &lhs, const ::proto::PublicTrade &rhs) const; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/include/proto-public-trade-converter.hpp b/src/serialization/include/proto-public-trade-converter.hpp new file mode 100644 index 00000000..ad2d7add --- /dev/null +++ b/src/serialization/include/proto-public-trade-converter.hpp @@ -0,0 +1,21 @@ +#pragma once + +#include "market.hpp" +#include "public-trade.pb.h" +#include "publictrade.hpp" + +namespace cct { + +::proto::PublicTrade ConvertPublicTradeToProto(const PublicTrade &publicTrade); + +class PublicTradeConverter { + public: + explicit PublicTradeConverter(Market market) : _market(market) {} + + PublicTrade operator()(const ::proto::PublicTrade &protoPublicTrade) const; + + private: + Market _market; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/include/proto-serializer.hpp b/src/serialization/include/proto-serializer.hpp new file mode 100644 index 00000000..017643da --- /dev/null +++ b/src/serialization/include/proto-serializer.hpp @@ -0,0 +1,264 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cct_exception.hpp" +#include "cct_log.hpp" +#include "cct_vector.hpp" +#include "durationstring.hpp" +#include "market-timestamp-set.hpp" +#include "market.hpp" +#include "proto-multiple-messages-handler.hpp" +#include "serialization-tools.hpp" +#include "stringhelpers.hpp" +#include "timedef.hpp" + +namespace cct { + +/// Class responsible to accumulate protobuf objects in memory and perform regular flushes of its data to the disk. +/// Data is accumulated by Market and will write to following files (from subPath): +/// 'BASECUR-QUOTECUR/YYYY/MM/DD/HH:00:00_HH:59:59.binpb' +/// +/// If you may push duplicated objects, you have to provide Comp and Equal types. +/// In this case, Equal must be consistent with Comp, and the first criteria of the comparison should be the timestamp +/// (ordered from oldest to youngest). +/// +/// You may not provide any Comp and Equal if by design you will not push duplicated data. +template +class ProtobufObjectsSerializer { + public: + /// Creates a new ProtobufObjectsSerializer. + /// @param marketTimestampSet the latest written timestamp for all markets to avoid writing duplicate entries between + /// coincenter restarts. + ProtobufObjectsSerializer(std::filesystem::path subPath, const MarketTimestampSet &marketTimestampSet, + int32_t nbObjectsPerMarketInMemory) + : _subPath(std::move(subPath)), _nbObjectsPerMarketInMemory(nbObjectsPerMarketInMemory) { + for (const auto &[market, timestamp] : marketTimestampSet) { + auto &lastWrittenObjectTimestamp = _marketDataMap[market].lastWrittenObjectTimestamp; + + lastWrittenObjectTimestamp = timestamp; + + // When program starts, we want to exclude equal timestamps to avoid writing objects that may have been written + // already from a previous run (the SortUnique will not protect us here) + ++lastWrittenObjectTimestamp; + } + } + + ProtobufObjectsSerializer(const ProtobufObjectsSerializer &) = delete; + ProtobufObjectsSerializer &operator=(const ProtobufObjectsSerializer &) = delete; + + ProtobufObjectsSerializer(ProtobufObjectsSerializer &&other) noexcept { swap(other); } + + ProtobufObjectsSerializer &operator=(ProtobufObjectsSerializer &&other) noexcept { + if (&other != this) { + swap(other); + } + return *this; + } + + /// At destruction of the serializer, we try to write all remaining objects in the buffer (as best effort mode). + ~ProtobufObjectsSerializer() { + try { + for (auto &[market, marketData] : _marketDataMap) { + writeOnDisk(market, marketData); + } + } catch (const std::exception &e) { + log::error("exception caught in writeOnDisk at ProtobufObjectsSerializer destruction: {}", e.what()); + } + } + + /// Pushes a new object into the serializer. + /// The new object is guaranteed to be written upon destruction of this serializer at the latest unless: + /// - its timestamp is older than the latest written timestamp of this market + /// - it has invalid data + template + void push(Market market, ProtobufObjectTypeU &&protoObj) { + if (!protoObj.has_unixtimestampinms()) { + throw exception("Attempt to push proto object without any timestamp"); + } + + auto &marketData = _marketDataMap[market]; + if (TimePoint{milliseconds{protoObj.unixtimestampinms()}} < marketData.lastWrittenObjectTimestamp) { + // do not push an object that has an older timestamp of the last written object + return; + } + + marketData.dataVector.push_back(std::forward(protoObj)); + + checkWriteOnDisk(market, marketData); + } + + void swap(ProtobufObjectsSerializer &rhs) noexcept { + _marketDataMap.swap(rhs._marketDataMap); + _subPath.swap(rhs._subPath); + std::swap(_nbObjectsPerMarketInMemory, rhs._nbObjectsPerMarketInMemory); + std::swap(_flushCounter, rhs._flushCounter); + } + + private: + using ProtobufObjectTypeVector = vector; + + struct MarketData { + ProtobufObjectTypeVector dataVector; + TimePoint lastWrittenObjectTimestamp; + }; + + void checkWriteOnDisk(Market market, MarketData &marketData) { + auto &dataVector = marketData.dataVector; + if (dataVector.size() == static_cast(_nbObjectsPerMarketInMemory)) { + writeOnDisk(market, marketData); + + // shrink_to_fit as vector will never grow-up larger than its current size + dataVector.shrink_to_fit(); + dataVector.clear(); + + checkPeriodicFlush(); + } + } + + void writeOnDisk(Market market, MarketData &marketData) { + auto &dataVector = marketData.dataVector; + if (dataVector.empty()) { + return; + } + + const auto nowTime = std::chrono::steady_clock::now(); + + SortUnique(dataVector); + + std::filesystem::path path; + + std::chrono::hours prevHourOfDay{-1}; + + ProtobufMessagesCompressedWriter protobufMessagesWriter; + + for (const auto &protobufObject : dataVector) { + checkOpenFile(market, protobufObject, prevHourOfDay, path, protobufMessagesWriter); + + protobufMessagesWriter.write(protobufObject); + } + + marketData.lastWrittenObjectTimestamp = TimePoint{milliseconds{dataVector.back().unixtimestampinms()}}; + + const auto nbElemsWritten = dataVector.size(); + + const auto steadyClockDuration = std::chrono::steady_clock::now() - nowTime; + const auto dur = std::chrono::duration_cast(steadyClockDuration); + + log::info("Serialized {} object(s) for {} data in {}, last in {}", nbElemsWritten, market, DurationToString(dur), + path.string()); + } + + // Periodic memory release to avoid possible leaks for long time running (if market data unused anymore for instance) + void checkPeriodicFlush() { + if (++_flushCounter != RehashThreshold) { + return; + } + + _flushCounter = 0; + + auto nowTime = Clock::now(); + + for (auto it = _marketDataMap.begin(); it != _marketDataMap.end();) { + if (it->second.lastWrittenObjectTimestamp + DurationType{static_cast(DurationValue)} < nowTime) { + // Unchanged data since a long time - write data if any, and clears the entry in the map + const Market market = it->first; + MarketData &marketData = it->second; + + writeOnDisk(market, marketData); + + log::info("Released {} protobuf objects for {}", marketData.dataVector.capacity(), market); + + it = _marketDataMap.erase(it); + } else { + ++it; + } + } + + _marketDataMap.rehash(_marketDataMap.size()); + } + + static void SortUnique(ProtobufObjectTypeVector &dataVector) { + static_assert((std::is_void_v && std::is_void_v) || (!std::is_void_v && !std::is_void_v)); + + if constexpr (std::is_void_v) { + // Sort by timestamp (required by 'writeOnDisk' algorithm) + std::ranges::sort(dataVector, [](const auto &lhs, const auto &rhs) { + return lhs.unixtimestampinms() < rhs.unixtimestampinms(); + }); + } else { + // We assume that timestamp is the first sorting criteria + std::ranges::sort(dataVector, Comp{}); + } + + // If duplicate elements are possible, remove them + if constexpr (!std::is_void_v) { + const auto [eraseIt1, eraseIt2] = std::ranges::unique(dataVector, Equal{}); + + dataVector.erase(eraseIt1, eraseIt2); + } + } + + void checkOpenFile(Market market, const ProtobufObjectType &protobufObject, std::chrono::hours &prevHourOfDay, + std::filesystem::path &path, + ProtobufMessagesCompressedWriter &protobufMessagesWriter) { + const TimePoint tp{milliseconds{protobufObject.unixtimestampinms()}}; + const auto hourOfDay = GetHourOfDay(tp); + + if (prevHourOfDay != hourOfDay) { + // open new outfile + setDirectory(market.str(), tp, path); + + std::filesystem::create_directories(path); + + path /= ComputeProtoFileName(std::chrono::duration_cast(hourOfDay).count()); + + std::ofstream ofs(path, std::ios_base::out | std::ios::binary | std::ios_base::app); + + if (!ofs.is_open()) { + throw exception("Cannot open the ofstream for writing to {}: {} (code {})", path.string(), std::strerror(errno), + errno); + } + + protobufMessagesWriter.open(std::move(ofs)); + prevHourOfDay = hourOfDay; + } + } + + static std::chrono::hours GetHourOfDay(TimePoint tp) { + const auto dp = std::chrono::floor(tp); + + return std::chrono::floor(tp - dp); + } + + void setDirectory(std::string_view marketStr, TimePoint tp, std::filesystem::path &path) { + const auto dp = std::chrono::floor(tp); + const std::chrono::year_month_day ymd{dp}; + + path = _subPath / marketStr; + path /= std::string_view(ToCharVector(static_cast(ymd.year()))); + path /= MonthStr(static_cast(ymd.month())); + path /= DayOfMonthStr(static_cast(ymd.day())); + } + + using MarketDataMap = std::unordered_map; + + MarketDataMap _marketDataMap; + std::filesystem::path _subPath; + int32_t _nbObjectsPerMarketInMemory; + int32_t _flushCounter{}; +}; + +} // namespace cct diff --git a/src/serialization/include/serialization-tools.hpp b/src/serialization/include/serialization-tools.hpp new file mode 100644 index 00000000..132cff73 --- /dev/null +++ b/src/serialization/include/serialization-tools.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include +#include + +namespace cct { + +/// Constructs a protobuf sub path containing the data directory, the exchange name and the object to be serialized. +std::filesystem::path ComputeProtoSubPath(std::string_view dataDir, std::string_view exchangeName, + std::string_view protobufObjectName); + +/// Converts a month integer (starting at 1) into a 2 fixed-sized string. +/// Example: +/// - 1 -> "01" +/// - 11 -> "11" +std::string_view MonthStr(int month); + +/// Converts a day of month int (starting at 1) into a 2 fixed-sized string. +/// Example: +/// - 1 -> "01" +/// - 11 -> "11" +std::string_view DayOfMonthStr(int dayOfMonth); + +/// From an hour of day in [0, 23], return the file name for a protobuf binary serialization file. +/// Example: +/// ComputeProtoFileName(4) -> "04:00:00_04:59:59.binpb" +std::string_view ComputeProtoFileName(int hourOfDay); + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/proto/market-order-book.proto b/src/serialization/proto/market-order-book.proto new file mode 100644 index 00000000..27efbba9 --- /dev/null +++ b/src/serialization/proto/market-order-book.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +package proto; + +message MarketOrderBook { + optional int64 unixTimestampInMs = 1; + optional int32 volumeNbDecimals = 2; + optional int32 priceNbDecimals = 3; + + message PricedVolume { + optional int64 price = 1; + optional int64 volume = 2; + } + + message OrderBook { + repeated PricedVolume asks = 1; + repeated PricedVolume bids = 2; + } + + optional OrderBook orderBook = 4; +} \ No newline at end of file diff --git a/src/serialization/proto/public-trade.proto b/src/serialization/proto/public-trade.proto new file mode 100644 index 00000000..25c50a2e --- /dev/null +++ b/src/serialization/proto/public-trade.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +package proto; + +enum TradeSide { + TRADE_UNSPECIFIED = 0; + TRADE_BUY = 1; + TRADE_SELL = 2; +} + +message PublicTrade { + optional int64 unixTimestampInMs = 1; + + optional int64 priceAmount = 2; + optional int64 volumeAmount = 3; + + optional int32 priceNbDecimals = 4; + optional int32 volumeNbDecimals = 5; + + TradeSide tradeSide = 6; +} \ No newline at end of file diff --git a/src/serialization/src/dummy-market-data-deserializer.cpp b/src/serialization/src/dummy-market-data-deserializer.cpp new file mode 100644 index 00000000..0fb1c9bf --- /dev/null +++ b/src/serialization/src/dummy-market-data-deserializer.cpp @@ -0,0 +1,31 @@ +#include "dummy-market-data-deserializer.hpp" + +#include + +#include "market-order-book-vector.hpp" +#include "market-timestamp-set.hpp" +#include "market.hpp" +#include "public-trade-vector.hpp" +#include "time-window.hpp" + +namespace cct { + +DummyMarketDataDeserializer::DummyMarketDataDeserializer([[maybe_unused]] std::string_view dataDir, + [[maybe_unused]] std::string_view exchangeName) {} + +MarketTimestampSet DummyMarketDataDeserializer::pullMarketOrderBooksMarkets([[maybe_unused]] TimeWindow timeWindow) { + return {}; +} + +MarketTimestampSet DummyMarketDataDeserializer::pullTradeMarkets([[maybe_unused]] TimeWindow timeWindow) { return {}; } + +MarketOrderBookVector DummyMarketDataDeserializer::pullMarketOrderBooks([[maybe_unused]] Market market, + [[maybe_unused]] TimeWindow timeWindow) { + return {}; +} + +PublicTradeVector DummyMarketDataDeserializer::pullTrades([[maybe_unused]] Market market, + [[maybe_unused]] TimeWindow timeWindow) { + return {}; +} +} // namespace cct \ No newline at end of file diff --git a/src/serialization/src/dummy-market-data-serializer.cpp b/src/serialization/src/dummy-market-data-serializer.cpp new file mode 100644 index 00000000..d3fd87f7 --- /dev/null +++ b/src/serialization/src/dummy-market-data-serializer.cpp @@ -0,0 +1,24 @@ +#include "dummy-market-data-serializer.hpp" + +#include +#include + +#include "market-timestamp-set.hpp" +#include "market.hpp" +#include "publictrade.hpp" + +namespace cct { + +class MarketOrderBook; + +DummyMarketDataSerializer::DummyMarketDataSerializer( + [[maybe_unused]] std::string_view dataDir, + [[maybe_unused]] const MarketTimestampSets &lastWrittenObjectsMarketTimestamp, + [[maybe_unused]] std::string_view exchangeName) {} + +void DummyMarketDataSerializer::push([[maybe_unused]] const MarketOrderBook &marketOrderBook) {} + +void DummyMarketDataSerializer::push([[maybe_unused]] Market market, + [[maybe_unused]] std::span publicTrades) {} + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/src/proto-market-data-deserializer.cpp b/src/serialization/src/proto-market-data-deserializer.cpp new file mode 100644 index 00000000..0289f011 --- /dev/null +++ b/src/serialization/src/proto-market-data-deserializer.cpp @@ -0,0 +1,33 @@ +#include "proto-market-data-deserializer.hpp" + +#include + +#include "market-order-book-vector.hpp" +#include "market-timestamp-set.hpp" +#include "market.hpp" +#include "proto-constants.hpp" +#include "public-trade-vector.hpp" +#include "serialization-tools.hpp" +#include "time-window.hpp" + +namespace cct { +ProtoMarketDataDeserializer::ProtoMarketDataDeserializer(std::string_view dataDir, std::string_view exchangeName) + : _marketOrderBookDeserializer(ComputeProtoSubPath(dataDir, exchangeName, kSubPathMarketOrderBooks)), + _publicTradeDeserializer(ComputeProtoSubPath(dataDir, exchangeName, kSubPathTrades)) {} + +MarketTimestampSet ProtoMarketDataDeserializer::pullMarketOrderBooksMarkets(TimeWindow timeWindow) { + return _publicTradeDeserializer.listMarkets(timeWindow); +} + +MarketTimestampSet ProtoMarketDataDeserializer::pullTradeMarkets(TimeWindow timeWindow) { + return _marketOrderBookDeserializer.listMarkets(timeWindow); +} + +MarketOrderBookVector ProtoMarketDataDeserializer::pullMarketOrderBooks(Market market, TimeWindow timeWindow) { + return _marketOrderBookDeserializer.loadMarket(market, timeWindow); +} + +PublicTradeVector ProtoMarketDataDeserializer::pullTrades(Market market, TimeWindow timeWindow) { + return _publicTradeDeserializer.loadMarket(market, timeWindow); +} +} // namespace cct \ No newline at end of file diff --git a/src/serialization/src/proto-market-data-serializer.cpp b/src/serialization/src/proto-market-data-serializer.cpp new file mode 100644 index 00000000..6ffa8779 --- /dev/null +++ b/src/serialization/src/proto-market-data-serializer.cpp @@ -0,0 +1,50 @@ +#include "proto-market-data-serializer.hpp" + +#include +#include + +#include "cct_log.hpp" +#include "market-timestamp-set.hpp" +#include "market.hpp" +#include "marketorderbook.hpp" +#include "proto-constants.hpp" +#include "proto-market-order-book-converter.hpp" +#include "proto-public-trade-converter.hpp" +#include "publictrade.hpp" +#include "serialization-tools.hpp" + +namespace cct { + +namespace { +constexpr auto kNbMarketOrderBookObjectsInMemory = 1000; +constexpr auto kNbTradeObjectsInMemory = 25000; +} // namespace + +ProtoMarketDataSerializer::ProtoMarketDataSerializer(std::string_view dataDir, + const MarketTimestampSets& lastWrittenObjectsMarketTimestamp, + std::string_view exchangeName) + : _marketOrderBookSerializer(ComputeProtoSubPath(dataDir, exchangeName, kSubPathMarketOrderBooks), + lastWrittenObjectsMarketTimestamp.orderBooksMarkets, + kNbMarketOrderBookObjectsInMemory), + _tradesSerializer(ComputeProtoSubPath(dataDir, exchangeName, kSubPathTrades), + lastWrittenObjectsMarketTimestamp.tradesMarkets, kNbTradeObjectsInMemory) {} + +void ProtoMarketDataSerializer::push(const MarketOrderBook& marketOrderBook) { + if (!marketOrderBook.isValid()) { + log::error("Do not serialize invalid market order book"); + return; + } + _marketOrderBookSerializer.push(marketOrderBook.market(), ConvertMarketOrderBookToProto(marketOrderBook)); +} + +void ProtoMarketDataSerializer::push(Market market, std::span publicTrades) { + for (const auto& publicTrade : publicTrades) { + if (!publicTrade.isValid()) { + log::error("Do not serialize invalid public trade"); + continue; + } + _tradesSerializer.push(market, ConvertPublicTradeToProto(publicTrade)); + } +} + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/src/proto-market-order-book-converter.cpp b/src/serialization/src/proto-market-order-book-converter.cpp new file mode 100644 index 00000000..67755a77 --- /dev/null +++ b/src/serialization/src/proto-market-order-book-converter.cpp @@ -0,0 +1,77 @@ +#include "proto-market-order-book-converter.hpp" + +#include +#include +#include + +#include "market-order-book.pb.h" +#include "marketorderbook.hpp" +#include "monetaryamount.hpp" +#include "timedef.hpp" +#include "volumeandpricenbdecimals.hpp" + +namespace cct { +::proto::MarketOrderBook ConvertMarketOrderBookToProto(const MarketOrderBook& marketOrderBook) { + ::proto::MarketOrderBook protoObj; + + const auto [volNbDecimals, priNbDecimals] = marketOrderBook.volAndPriNbDecimals(); + const auto unixTimestampInMs = TimestampToMillisecondsSinceEpoch(marketOrderBook.time()); + + protoObj.set_unixtimestampinms(unixTimestampInMs); + protoObj.set_volumenbdecimals(volNbDecimals); + protoObj.set_pricenbdecimals(priNbDecimals); + + auto& orderBook = *protoObj.mutable_orderbook(); + + const int nbBids = marketOrderBook.nbBidPrices(); + for (int bidPos = 1; bidPos <= nbBids; ++bidPos) { + const auto [volume, price] = marketOrderBook[-bidPos]; + auto& pricedVolume = *orderBook.add_bids(); + + pricedVolume.set_volume(volume.amount(volNbDecimals).value()); + pricedVolume.set_price(price.amount(priNbDecimals).value()); + } + + const int nbAsks = marketOrderBook.nbAskPrices(); + for (int askPos = 1; askPos <= nbAsks; ++askPos) { + const auto [volume, price] = marketOrderBook[askPos]; + auto& pricedVolume = *orderBook.add_asks(); + + pricedVolume.set_volume(volume.amount(volNbDecimals).value()); + pricedVolume.set_price(price.amount(priNbDecimals).value()); + } + + return protoObj; +} + +MarketOrderBook MarketOrderBookConverter::operator()(const ::proto::MarketOrderBook& marketOrderBookTimedData) { + const TimePoint timeStamp(milliseconds(marketOrderBookTimedData.unixtimestampinms())); + const VolAndPriNbDecimals volAndPriNbDecimals(marketOrderBookTimedData.volumenbdecimals(), + marketOrderBookTimedData.pricenbdecimals()); + + const auto& bids = marketOrderBookTimedData.orderbook().bids(); + const auto& asks = marketOrderBookTimedData.orderbook().asks(); + const int32_t lowestAskPricePos = static_cast(bids.size()); + const int32_t highestBidPricePos = lowestAskPricePos - 1; + + // We directly construct the MarketOrderBook here - we trust the protobuf data (it should have been written from a + // valid MarketOrderBook at the source) + // Possible optimization - allocate in a reusable arena of memory instead of allocating a new buffer for each new + // object. + MarketOrderBook::AmountPriceVector orders; + + orders.reserve(bids.size() + asks.size()); + + for (const auto& bid : std::ranges::reverse_view(bids)) { + orders.emplace_back(bid.volume(), bid.price()); + } + + for (const auto& ask : asks) { + orders.emplace_back(-ask.volume(), ask.price()); + } + + return MarketOrderBook{timeStamp, _market, std::move(orders), + highestBidPricePos, lowestAskPricePos, volAndPriNbDecimals}; +} + +} // namespace cct diff --git a/src/serialization/src/proto-public-trade-compare.cpp b/src/serialization/src/proto-public-trade-compare.cpp new file mode 100644 index 00000000..9049a7ec --- /dev/null +++ b/src/serialization/src/proto-public-trade-compare.cpp @@ -0,0 +1,48 @@ +#include "proto-public-trade-compare.hpp" + +#include "currencycode.hpp" +#include "monetaryamount.hpp" +#include "public-trade.pb.h" + +namespace cct { + +namespace { +MonetaryAmount PriceMonetaryAmount(const ::proto::PublicTrade& obj) { + return MonetaryAmount(obj.priceamount(), CurrencyCode{}, obj.pricenbdecimals()); +} + +MonetaryAmount VolumeMonetaryAmount(const ::proto::PublicTrade& obj) { + return MonetaryAmount(obj.volumeamount(), CurrencyCode{}, obj.volumenbdecimals()); +} +} // namespace + +bool ProtoPublicTradeComp::operator()(const ::proto::PublicTrade& lhs, const ::proto::PublicTrade& rhs) const { + if (lhs.unixtimestampinms() != rhs.unixtimestampinms()) { + return lhs.unixtimestampinms() < rhs.unixtimestampinms(); + } + + MonetaryAmount lhsAmount = VolumeMonetaryAmount(lhs); + MonetaryAmount rhsAmount = VolumeMonetaryAmount(rhs); + if (lhsAmount != rhsAmount) { + return lhsAmount < rhsAmount; + } + + MonetaryAmount lhsPrice = PriceMonetaryAmount(lhs); + MonetaryAmount rhsPrice = PriceMonetaryAmount(rhs); + if (lhsPrice != rhsPrice) { + return lhsPrice < rhsPrice; + } + + if (lhs.tradeside() != rhs.tradeside()) { + return lhs.tradeside() < rhs.tradeside(); + } + + return false; +} + +bool ProtoPublicTradeEqual::operator()(const ::proto::PublicTrade& lhs, const ::proto::PublicTrade& rhs) const { + return lhs.unixtimestampinms() == rhs.unixtimestampinms() && VolumeMonetaryAmount(lhs) == VolumeMonetaryAmount(rhs) && + PriceMonetaryAmount(lhs) == PriceMonetaryAmount(rhs) && lhs.tradeside() == rhs.tradeside(); +} + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/src/proto-public-trade-converter.cpp b/src/serialization/src/proto-public-trade-converter.cpp new file mode 100644 index 00000000..298ffaa0 --- /dev/null +++ b/src/serialization/src/proto-public-trade-converter.cpp @@ -0,0 +1,63 @@ +#include "proto-public-trade-converter.hpp" + +#include "monetaryamount.hpp" +#include "public-trade.pb.h" +#include "publictrade.hpp" +#include "timedef.hpp" +#include "tradeside.hpp" +#include "unreachable.hpp" + +namespace cct { + +namespace { +::proto::TradeSide ConvertTradeSide(TradeSide tradeSide) { + switch (tradeSide) { + case TradeSide::kBuy: + return ::proto::TRADE_BUY; + case TradeSide::kSell: + return ::proto::TRADE_SELL; + default: + unreachable(); + } +} + +TradeSide ConvertTradeSide(::proto::TradeSide tradeSide) { + switch (tradeSide) { + case ::proto::TRADE_BUY: + return TradeSide::kBuy; + case ::proto::TRADE_SELL: + return TradeSide::kSell; + default: + unreachable(); + } +} + +} // namespace + +::proto::PublicTrade ConvertPublicTradeToProto(const PublicTrade &publicTrade) { + ::proto::PublicTrade protoObj; + + protoObj.set_unixtimestampinms(TimestampToMillisecondsSinceEpoch(publicTrade.time())); + + const auto price = publicTrade.price(); + protoObj.set_priceamount(price.amount()); + protoObj.set_pricenbdecimals(price.nbDecimals()); + + const auto volume = publicTrade.amount(); + protoObj.set_volumeamount(volume.amount()); + protoObj.set_volumenbdecimals(volume.nbDecimals()); + + protoObj.set_tradeside(ConvertTradeSide(publicTrade.side())); + + return protoObj; +} + +PublicTrade PublicTradeConverter::operator()(const ::proto::PublicTrade &protoPublicTrade) const { + const MonetaryAmount amount(protoPublicTrade.volumeamount(), _market.base(), protoPublicTrade.volumenbdecimals()); + const MonetaryAmount price(protoPublicTrade.priceamount(), _market.quote(), protoPublicTrade.pricenbdecimals()); + const TimePoint timeStamp(milliseconds(protoPublicTrade.unixtimestampinms())); + + return {ConvertTradeSide(protoPublicTrade.tradeside()), amount, price, timeStamp}; +} + +} // namespace cct diff --git a/src/serialization/src/serialization-tools.cpp b/src/serialization/src/serialization-tools.cpp new file mode 100644 index 00000000..5dfc71a9 --- /dev/null +++ b/src/serialization/src/serialization-tools.cpp @@ -0,0 +1,86 @@ +#include "serialization-tools.hpp" + +#include +#include +#include +#include + +#include "proto-constants.hpp" + +namespace cct { + +std::filesystem::path ComputeProtoSubPath(std::string_view dataDir, std::string_view exchangeName, + std::string_view protobufObjectName) { + std::filesystem::path ret(dataDir); + + ret /= "serialized"; + ret /= protobufObjectName; + ret /= exchangeName; + + return ret; +} + +namespace { + +consteval auto BuildIntLessThan31Strings() { + // 31 days + constexpr auto kNbMaxUnit = 31; + + std::array, kNbMaxUnit> ret; + + for (auto unit = 1; unit <= kNbMaxUnit; ++unit) { + ret[unit - 1][0] = (unit / 10) + '0'; + ret[unit - 1][1] = (unit % 10) + '0'; + } + + return ret; +} + +constexpr auto kIntLessThan31Strings = BuildIntLessThan31Strings(); + +} // namespace + +std::string_view MonthStr(int month) { return std::string_view{kIntLessThan31Strings[month - 1]}; } + +std::string_view DayOfMonthStr(int dayOfMonth) { return std::string_view{kIntLessThan31Strings[dayOfMonth - 1]}; } + +namespace { + +consteval auto BuildBinProtoFileNames() { + // We use '-' instead of ':' as hour / minutes separator as some file systems do not accept this character in a + // filename (Windows for instance) + constexpr std::string_view kBinProtoFileHourBegPart = "-00-00_"; + constexpr std::string_view kBinProtoFileHourEndPart = "-59-59"; + + using ProtoFileNameBuffer = + std::array(2 * 2)>; + + constexpr auto kNbHourInDay = 24; + + std::array ret; + + for (auto hourOfDay = 0; hourOfDay < kNbHourInDay; ++hourOfDay) { + std::array hourStr = {static_cast((hourOfDay / 10) + '0'), static_cast((hourOfDay % 10) + '0')}; + + auto it = ret[hourOfDay].begin(); + + it = std::ranges::copy(hourStr, it).out; + it = std::ranges::copy(kBinProtoFileHourBegPart, it).out; + it = std::ranges::copy(hourStr, it).out; + it = std::ranges::copy(kBinProtoFileHourEndPart, it).out; + it = std::ranges::copy(kBinProtobufExtension, it).out; + } + + return ret; +} + +} // namespace + +std::string_view ComputeProtoFileName(int hourOfDay) { + static constexpr auto kBinProtoFileNames = BuildBinProtoFileNames(); + + return std::string_view{kBinProtoFileNames[hourOfDay]}; +} + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/test/continuous-iterator_test.cpp b/src/serialization/test/continuous-iterator_test.cpp new file mode 100644 index 00000000..b6eaa089 --- /dev/null +++ b/src/serialization/test/continuous-iterator_test.cpp @@ -0,0 +1,41 @@ +#include "continuous-iterator.hpp" + +#include + +namespace cct { + +TEST(ContinuousIterator, UniqueElement) { + ContinuousIterator it(1, 1); + + EXPECT_TRUE(it.hasNext()); + EXPECT_EQ(it.next(), 1); + EXPECT_FALSE(it.hasNext()); +} + +TEST(ContinuousIterator, SeveralElements) { + ContinuousIterator it(1, 3); + + EXPECT_TRUE(it.hasNext()); + EXPECT_EQ(it.next(), 1); + EXPECT_TRUE(it.hasNext()); + EXPECT_EQ(it.next(), 2); + EXPECT_TRUE(it.hasNext()); + EXPECT_EQ(it.next(), 3); + EXPECT_FALSE(it.hasNext()); +} + +TEST(ContinuousIterator, Reverse) { + ContinuousIterator it(1, -2); + + EXPECT_TRUE(it.hasNext()); + EXPECT_EQ(it.next(), 1); + EXPECT_TRUE(it.hasNext()); + EXPECT_EQ(it.next(), 0); + EXPECT_TRUE(it.hasNext()); + EXPECT_EQ(it.next(), -1); + EXPECT_TRUE(it.hasNext()); + EXPECT_EQ(it.next(), -2); + EXPECT_FALSE(it.hasNext()); +} + +} // namespace cct diff --git a/src/serialization/test/proto-market-order-book-converter_test.cpp b/src/serialization/test/proto-market-order-book-converter_test.cpp new file mode 100644 index 00000000..5a9a47b6 --- /dev/null +++ b/src/serialization/test/proto-market-order-book-converter_test.cpp @@ -0,0 +1,115 @@ +#include "proto-market-order-book-converter.hpp" + +#include + +#include + +#include "amount-price.hpp" +#include "marketorderbook.hpp" +#include "monetaryamount.hpp" +#include "order-book-line.hpp" + +namespace cct { + +// TODO: factorize duplicated code from marketorderbook_test +namespace { +using AmountAtPriceVec = MarketOrderBook::AmountPerPriceVec; +} // namespace + +MarketOrderBookLines CreateMarketOrderBookLines(std::initializer_list init) { + MarketOrderBookLines marketOrderBookLines; + marketOrderBookLines.reserve(init.size()); + + for (const auto &orderBookLine : init) { + if (orderBookLine.amount() < 0) { + marketOrderBookLines.pushAsk(-orderBookLine.amount(), orderBookLine.price()); + } else { + marketOrderBookLines.pushBid(orderBookLine.amount(), orderBookLine.price()); + } + } + + return marketOrderBookLines; +} + +constexpr bool operator==(const AmountPrice &lhs, const AmountPrice &rhs) { + return lhs.amount == rhs.amount && lhs.price == rhs.price; +} + +class ProtoMarketOrderBookTest : public ::testing::Test { + protected: + TimePoint time; + Market market{"APM", "KRW"}; + MarketOrderBook marketOrderBook{ + time, market, + CreateMarketOrderBookLines( + {OrderBookLine(MonetaryAmount("1991.3922", "APM"), MonetaryAmount("57.8", "KRW"), OrderBookLine::Type::kAsk), + OrderBookLine(MonetaryAmount("90184.3951", "APM"), MonetaryAmount("57.81", "KRW"), + OrderBookLine::Type::kAsk), + OrderBookLine(MonetaryAmount("91.1713", "APM"), MonetaryAmount("57.84", "KRW"), OrderBookLine::Type::kAsk), + OrderBookLine(MonetaryAmount("41.0131", "APM"), MonetaryAmount("57.9", "KRW"), OrderBookLine::Type::kAsk), + OrderBookLine(MonetaryAmount("33.5081914157147802", "APM"), MonetaryAmount("57.78", "KRW"), + OrderBookLine::Type::kAsk), + OrderBookLine(MonetaryAmount("3890.879", "APM"), MonetaryAmount("57.19", "KRW"), OrderBookLine::Type::kBid), + OrderBookLine(MonetaryAmount("14", "APM"), MonetaryAmount("57.18", "KRW"), OrderBookLine::Type::kBid), + OrderBookLine(MonetaryAmount("14", "APM"), MonetaryAmount("57.17", "KRW"), OrderBookLine::Type::kBid), + OrderBookLine(MonetaryAmount("3848.8453", "APM"), MonetaryAmount("57.16", "KRW"), + OrderBookLine::Type::kBid)})}; + + MarketOrderBookConverter marketOrderBookConverter{market}; +}; + +TEST_F(ProtoMarketOrderBookTest, Serialization) { + const auto protoObj = ConvertMarketOrderBookToProto(marketOrderBook); + const auto [volNbDecimals, priNbDecimals] = marketOrderBook.volAndPriNbDecimals(); + + EXPECT_EQ(TimePoint{milliseconds{protoObj.unixtimestampinms()}}, marketOrderBook.time()); + EXPECT_EQ(protoObj.volumenbdecimals(), volNbDecimals); + EXPECT_EQ(protoObj.pricenbdecimals(), priNbDecimals); + + ASSERT_TRUE(protoObj.has_orderbook()); + ASSERT_EQ(protoObj.orderbook().asks_size(), 5U); + + const auto &asks = protoObj.orderbook().asks(); + + EXPECT_EQ(asks[0].volume(), 335081914157147); + EXPECT_EQ(asks[0].price(), 577800000000000000); + + EXPECT_EQ(asks[1].volume(), 19913922000000000); + EXPECT_EQ(asks[1].price(), 578000000000000000); + + EXPECT_EQ(asks[2].volume(), 901843951000000000); + EXPECT_EQ(asks[2].price(), 578100000000000000); + + EXPECT_EQ(asks[3].volume(), 911713000000000); + EXPECT_EQ(asks[3].price(), 578400000000000000); + + EXPECT_EQ(asks[4].volume(), 410131000000000); + EXPECT_EQ(asks[4].price(), 579000000000000000); + + ASSERT_EQ(protoObj.orderbook().bids_size(), 4U); + + const auto &bids = protoObj.orderbook().bids(); + + EXPECT_EQ(bids[0].volume(), 38908790000000000); + EXPECT_EQ(bids[0].price(), 571900000000000000); + + EXPECT_EQ(bids[1].volume(), 140000000000000); + EXPECT_EQ(bids[1].price(), 571800000000000000); + + EXPECT_EQ(bids[2].volume(), 140000000000000); + EXPECT_EQ(bids[2].price(), 571700000000000000); + + EXPECT_EQ(bids[3].volume(), 38488453000000000); + EXPECT_EQ(bids[3].price(), 571600000000000000); +} + +TEST_F(ProtoMarketOrderBookTest, SerializeThenDeserializeShouldGiveSameObject) { + const auto protoObj = ConvertMarketOrderBookToProto(marketOrderBook); + + const auto marketOrderBookConvertedBack = marketOrderBookConverter(protoObj); + + EXPECT_TRUE(marketOrderBookConvertedBack.isValid()); + + EXPECT_EQ(marketOrderBook, marketOrderBookConvertedBack); +} +} // namespace cct \ No newline at end of file diff --git a/src/serialization/test/proto-multiple-messages-handler_test.cpp b/src/serialization/test/proto-multiple-messages-handler_test.cpp new file mode 100644 index 00000000..88279c0f --- /dev/null +++ b/src/serialization/test/proto-multiple-messages-handler_test.cpp @@ -0,0 +1,121 @@ +#include "proto-multiple-messages-handler.hpp" + +#include + +#include + +#include "cct_exception.hpp" +#include "proto-public-trade-converter.hpp" +#include "proto-test-data.hpp" +#include "public-trade.pb.h" +#include "publictrade.hpp" + +namespace cct { + +class ProtobufMessagesTest : public ProtobufBaseDataTest { + protected: + PublicTradeConverter publicTradeConverter{mk1}; + + ProtobufMessagesCompressedWriter writer; +}; + +TEST_F(ProtobufMessagesTest, DefaultConstruction) {} + +TEST_F(ProtobufMessagesTest, OpenShouldBeCalledBeforeWrite) { EXPECT_THROW(writer.write(td1), exception); } + +TEST_F(ProtobufMessagesTest, WriteReadSingle) { + writer.open(std::stringstream{}); + writer.write(td1); + + std::stringstream ss = writer.flush(); + + ProtobufMessageCompressedReaderIterator reader{ss}; + + int nbObjectsRead = 0; + + while (reader.hasNext()) { + auto nextObj = reader.next<::proto::PublicTrade>(); + PublicTrade pt = publicTradeConverter(nextObj); + + EXPECT_EQ(pt, pt1); + ++nbObjectsRead; + } + + EXPECT_EQ(nbObjectsRead, 1); +} + +TEST_F(ProtobufMessagesTest, WriteRead2Flushes) { + writer.open(std::stringstream{}); + writer.write(td1); + + std::stringstream ss1 = writer.flush(); + + writer.open(std::stringstream{}); + writer.write(td2); + + std::stringstream ss2 = writer.flush(); + + ProtobufMessageCompressedReaderIterator reader1{ss1}; + + int nbObjectsRead = 0; + + while (reader1.hasNext()) { + auto nextObj = reader1.next<::proto::PublicTrade>(); + PublicTrade pt = publicTradeConverter(nextObj); + + EXPECT_EQ(pt, pt1); + ++nbObjectsRead; + } + + EXPECT_EQ(nbObjectsRead, 1); + + ProtobufMessageCompressedReaderIterator reader2{ss2}; + + while (reader2.hasNext()) { + auto nextObj = reader2.next<::proto::PublicTrade>(); + PublicTrade pt = publicTradeConverter(nextObj); + + EXPECT_EQ(pt, pt2); + ++nbObjectsRead; + } + + EXPECT_EQ(nbObjectsRead, 2); +} + +TEST_F(ProtobufMessagesTest, WriteReadSeveral) { + writer.open(std::stringstream{}); + writer.write(td1); + writer.write(td2); + writer.write(td3); + + std::stringstream ss = writer.flush(); + + ProtobufMessageCompressedReaderIterator reader{ss}; + + int nbObjectsRead = 0; + + while (reader.hasNext()) { + auto nextObj = reader.next<::proto::PublicTrade>(); + PublicTrade pt = publicTradeConverter(nextObj); + + switch (nbObjectsRead) { + case 0: + EXPECT_EQ(pt, pt1); + break; + case 1: + EXPECT_EQ(pt, pt2); + break; + case 2: + EXPECT_EQ(pt, pt3); + break; + default: + break; + } + + ++nbObjectsRead; + } + + EXPECT_EQ(nbObjectsRead, 3); +} + +} // namespace cct diff --git a/src/serialization/test/proto-public-trade-converter_test.cpp b/src/serialization/test/proto-public-trade-converter_test.cpp new file mode 100644 index 00000000..3c9a8f72 --- /dev/null +++ b/src/serialization/test/proto-public-trade-converter_test.cpp @@ -0,0 +1,31 @@ +#include "proto-public-trade-converter.hpp" + +#include + +#include +#include + +#include "market.hpp" +#include "monetaryamount.hpp" +#include "publictrade.hpp" +#include "timedef.hpp" +#include "tradeside.hpp" + +namespace cct { + +class ProtoPublicTradeTest : public ::testing::Test { + protected: + TimePoint tp{milliseconds{std::numeric_limits::max() / 10000000}}; + Market market{"ETH", "USDT"}; + PublicTrade pt{TradeSide::kBuy, MonetaryAmount{"0.13", "ETH"}, MonetaryAmount{"1500.5", "USDT"}, tp}; + PublicTradeConverter publicTradeConverter{market}; +}; + +TEST_F(ProtoPublicTradeTest, SerializeThenDeserializeShouldGiveSameObject) { + const auto protoObj = ConvertPublicTradeToProto(pt); + + const auto objBack = publicTradeConverter(protoObj); + + EXPECT_EQ(pt, objBack); +} +} // namespace cct \ No newline at end of file diff --git a/src/serialization/test/proto-serialization-and-deserialization_test.cpp b/src/serialization/test/proto-serialization-and-deserialization_test.cpp new file mode 100644 index 00000000..7e01ec84 --- /dev/null +++ b/src/serialization/test/proto-serialization-and-deserialization_test.cpp @@ -0,0 +1,622 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "market-timestamp-set.hpp" +#include "market-timestamp.hpp" +#include "monetaryamount.hpp" +#include "proto-deserializer.hpp" +#include "proto-public-trade-compare.hpp" +#include "proto-serializer.hpp" +#include "proto-test-data.hpp" +#include "public-trade-vector.hpp" +#include "public-trade.pb.h" +#include "publictrade.hpp" +#include "time-window.hpp" +#include "timedef.hpp" + +namespace cct { + +namespace { +constexpr auto RandStr() { + std::array str; + + std::ranges::generate(str, []() { + static constexpr char kCharSet[] = + "0123456789" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz"; + static constexpr size_t kMaxIdx = (sizeof(kCharSet) - 1); + return kCharSet[rand() % kMaxIdx]; + }); + + return str; +} +} // namespace + +class ProtobufSerializerDeserializerTest : public ProtobufBaseDataTest { + protected: + void TearDown() override { + std::filesystem::remove_all(subPath1); + std::filesystem::remove_all(subPath2); + } + + std::filesystem::path subPath1 = std::filesystem::temp_directory_path() / std::string_view(RandStr()); + std::filesystem::path subPath2 = std::filesystem::temp_directory_path() / std::string_view(RandStr()); + + static constexpr int32_t kSmallRehashThreshold = 3; + + using DurationTypeFlush = milliseconds; + static constexpr int32_t kFlushNbMillis = 1; + + int32_t nbTradesPerMarketInMemory{100}; + + using Serializer = ProtobufObjectsSerializer<::proto::PublicTrade, ProtoPublicTradeComp, ProtoPublicTradeEqual, + kSmallRehashThreshold, DurationTypeFlush, kFlushNbMillis>; + using Deserializer = ProtobufObjectsDeserializer<::proto::PublicTrade, PublicTradeConverter>; + + Serializer createSerializer(const MarketTimestampSet &marketTimestampSet = MarketTimestampSet{}) { + return Serializer{subPath1, marketTimestampSet, nbTradesPerMarketInMemory}; + } + Deserializer createDeserializer() { return Deserializer{subPath1}; } + + void serializeSomeObjects(Serializer &serializer) { + // push two times same object (should not be duplicated during writing) + serializer.push(mk1, td2); + serializer.push(mk1, td2); + + // Even if older object, should be pushed as well as serializer has not written any objects yet (they should be + // ordered internally before writes) + serializer.push(mk1, td1); + + serializer.push(mk1, td3); + serializer.push(mk3, td4); + serializer.push(mk4, td5); + + serializer.push(mk1, td9); + + // To force a write and make sure that serializer writes in append mode on a same file + for (decltype(nbTradesPerMarketInMemory) pushPos = 0; pushPos < nbTradesPerMarketInMemory; ++pushPos) { + serializer.push(mk5, td10); + } + + std::this_thread::sleep_for(milliseconds{2}); + + serializer.push(mk5, td11); + + serializer.push(mk7, td7); + + Serializer anotherSerializer{subPath2, MarketTimestampSet{}, nbTradesPerMarketInMemory}; + + serializer.swap(anotherSerializer); + + // Should not be pushed to 'subPath' + serializer.push(mk6, td6); + } +}; + +TEST_F(ProtobufSerializerDeserializerTest, SerializeThenDeserializeSomeObjects) { + { + auto serializer = createSerializer(); + serializeSomeObjects(serializer); + } + + // make sure serializer writes all at destruction + + const std::filesystem::path kExpectedFiles[] = { + subPath2 / std::string_view{mk6.str()} / "2012" / "12" / "24" / ComputeProtoFileName(19), + subPath1 / std::string_view{mk7.str()} / "2014" / "04" / "14" / ComputeProtoFileName(13), + subPath1 / std::string_view{mk1.str()} / "2006" / "07" / "14" / ComputeProtoFileName(23), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "23" / ComputeProtoFileName(7), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "25" / ComputeProtoFileName(4), + subPath1 / std::string_view{mk1.str()} / "2012" / "05" / "11" / ComputeProtoFileName(7), + subPath1 / std::string_view{mk3.str()} / "1999" / "03" / "25" / ComputeProtoFileName(4), + subPath1 / std::string_view{mk4.str()} / "2013" / "08" / "16" / ComputeProtoFileName(3), + subPath1 / std::string_view{mk5.str()} / "2014" / "12" / "19" / ComputeProtoFileName(9)}; + + auto isFilePresent = [](const auto &fileName) { return std::filesystem::exists(fileName); }; + + EXPECT_TRUE(std::ranges::all_of(kExpectedFiles, isFilePresent)); + + auto deserializer = createDeserializer(); + auto marketTimestampSet = deserializer.listMarkets(timeWindowAll); + + EXPECT_EQ(marketTimestampSet, + MarketTimestampSet({MarketTimestamp(mk7, tp8), MarketTimestamp(mk1, tp5), MarketTimestamp(mk3, tp1), + MarketTimestamp(mk4, tp7), MarketTimestamp(mk5, tp10)})); + + // Should not serialize again as timestamps are set to last written + { + auto serializer = createSerializer(marketTimestampSet); + serializeSomeObjects(serializer); + } + + EXPECT_TRUE(deserializer.loadMarket(mk1, timeWindow79).empty()); + EXPECT_EQ(deserializer.loadMarket(mk1, timeWindowAll), vector({pt1, pt2, pt3, pt9})); + EXPECT_EQ(deserializer.loadMarket(mk4, timeWindowAll), vector({pt5})); + EXPECT_TRUE(deserializer.loadMarket(mk4, timeWindow14).empty()); + EXPECT_EQ(deserializer.loadMarket(mk7, timeWindowAll), vector({pt7})); + EXPECT_EQ(deserializer.loadMarket(mk7, timeWindow79), vector({pt7})); + EXPECT_TRUE(deserializer.loadMarket(Market{"UNK", "OTH"}, timeWindowAll).empty()); + EXPECT_EQ(deserializer.loadMarket(mk5, timeWindowAll), vector({pt10, pt11})); +} + +TEST_F(ProtobufSerializerDeserializerTest, ManySerializationsDifferentHoursOfDay) { + static const TimePoint kTimePoints[] = {tp1, tp2}; + static const Market kMarkets[] = {mk1, mk4}; + static constexpr Duration kDurationRange = std::chrono::weeks(2); + static constexpr Duration kDurationStep = std::chrono::minutes(199); + + std::map pushedPublicTrades; + + { + auto serializer = createSerializer(); + + for (Market market : kMarkets) { + for (TimePoint tp : kTimePoints) { + for (auto ts = tp; ts < tp + kDurationRange; ts += kDurationStep) { + TradeSide side = tp == tp1 ? TradeSide::kBuy : TradeSide::kSell; + MonetaryAmount amount{"0.13", market.base()}; + MonetaryAmount price{"1500.5", market.quote()}; + PublicTrade pt{side, amount, price, ts}; + + pushedPublicTrades[market].push_back(pt); + serializer.push(market, ConvertPublicTradeToProto(pt)); + } + + std::this_thread::sleep_for(milliseconds{2}); + } + } + } + + const std::filesystem::path kExpectedFiles[] = { + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "25" / ComputeProtoFileName(4), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "25" / ComputeProtoFileName(8), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "25" / ComputeProtoFileName(11), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "25" / ComputeProtoFileName(14), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "25" / ComputeProtoFileName(18), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "25" / ComputeProtoFileName(21), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "26" / ComputeProtoFileName(0), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "26" / ComputeProtoFileName(3), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "26" / ComputeProtoFileName(7), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "26" / ComputeProtoFileName(10), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "26" / ComputeProtoFileName(13), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "26" / ComputeProtoFileName(17), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "26" / ComputeProtoFileName(20), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "26" / ComputeProtoFileName(23), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "27" / ComputeProtoFileName(3), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "27" / ComputeProtoFileName(6), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "27" / ComputeProtoFileName(9), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "27" / ComputeProtoFileName(13), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "27" / ComputeProtoFileName(16), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "27" / ComputeProtoFileName(19), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "27" / ComputeProtoFileName(23), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "28" / ComputeProtoFileName(2), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "28" / ComputeProtoFileName(5), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "28" / ComputeProtoFileName(9), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "28" / ComputeProtoFileName(12), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "28" / ComputeProtoFileName(15), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "28" / ComputeProtoFileName(19), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "28" / ComputeProtoFileName(22), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "29" / ComputeProtoFileName(1), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "29" / ComputeProtoFileName(4), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "29" / ComputeProtoFileName(8), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "29" / ComputeProtoFileName(11), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "29" / ComputeProtoFileName(14), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "29" / ComputeProtoFileName(18), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "29" / ComputeProtoFileName(21), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "30" / ComputeProtoFileName(0), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "30" / ComputeProtoFileName(4), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "30" / ComputeProtoFileName(7), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "30" / ComputeProtoFileName(10), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "30" / ComputeProtoFileName(14), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "30" / ComputeProtoFileName(17), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "30" / ComputeProtoFileName(20), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "31" / ComputeProtoFileName(0), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "31" / ComputeProtoFileName(3), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "31" / ComputeProtoFileName(6), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "31" / ComputeProtoFileName(10), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "31" / ComputeProtoFileName(13), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "31" / ComputeProtoFileName(16), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "31" / ComputeProtoFileName(19), + subPath1 / std::string_view{mk1.str()} / "1999" / "03" / "31" / ComputeProtoFileName(23), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "01" / ComputeProtoFileName(2), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "01" / ComputeProtoFileName(5), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "01" / ComputeProtoFileName(9), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "01" / ComputeProtoFileName(12), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "01" / ComputeProtoFileName(15), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "01" / ComputeProtoFileName(19), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "01" / ComputeProtoFileName(22), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "02" / ComputeProtoFileName(1), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "02" / ComputeProtoFileName(5), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "02" / ComputeProtoFileName(8), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "02" / ComputeProtoFileName(11), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "02" / ComputeProtoFileName(15), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "02" / ComputeProtoFileName(18), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "02" / ComputeProtoFileName(21), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "03" / ComputeProtoFileName(1), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "03" / ComputeProtoFileName(4), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "03" / ComputeProtoFileName(7), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "03" / ComputeProtoFileName(10), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "03" / ComputeProtoFileName(14), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "03" / ComputeProtoFileName(17), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "03" / ComputeProtoFileName(20), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "04" / ComputeProtoFileName(0), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "04" / ComputeProtoFileName(3), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "04" / ComputeProtoFileName(6), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "04" / ComputeProtoFileName(10), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "04" / ComputeProtoFileName(13), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "04" / ComputeProtoFileName(16), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "04" / ComputeProtoFileName(20), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "04" / ComputeProtoFileName(23), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "05" / ComputeProtoFileName(2), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "05" / ComputeProtoFileName(6), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "05" / ComputeProtoFileName(9), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "05" / ComputeProtoFileName(12), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "05" / ComputeProtoFileName(16), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "05" / ComputeProtoFileName(19), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "05" / ComputeProtoFileName(22), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "06" / ComputeProtoFileName(2), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "06" / ComputeProtoFileName(5), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "06" / ComputeProtoFileName(8), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "06" / ComputeProtoFileName(11), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "06" / ComputeProtoFileName(15), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "06" / ComputeProtoFileName(18), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "06" / ComputeProtoFileName(21), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "07" / ComputeProtoFileName(1), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "07" / ComputeProtoFileName(4), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "07" / ComputeProtoFileName(7), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "07" / ComputeProtoFileName(11), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "07" / ComputeProtoFileName(14), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "07" / ComputeProtoFileName(17), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "07" / ComputeProtoFileName(21), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "08" / ComputeProtoFileName(0), + subPath1 / std::string_view{mk1.str()} / "1999" / "04" / "08" / ComputeProtoFileName(3), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "23" / ComputeProtoFileName(7), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "23" / ComputeProtoFileName(11), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "23" / ComputeProtoFileName(14), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "23" / ComputeProtoFileName(17), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "23" / ComputeProtoFileName(21), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "24" / ComputeProtoFileName(0), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "24" / ComputeProtoFileName(3), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "24" / ComputeProtoFileName(7), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "24" / ComputeProtoFileName(10), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "24" / ComputeProtoFileName(13), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "24" / ComputeProtoFileName(17), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "24" / ComputeProtoFileName(20), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "24" / ComputeProtoFileName(23), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "25" / ComputeProtoFileName(3), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "25" / ComputeProtoFileName(6), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "25" / ComputeProtoFileName(9), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "25" / ComputeProtoFileName(13), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "25" / ComputeProtoFileName(16), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "25" / ComputeProtoFileName(19), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "25" / ComputeProtoFileName(22), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "26" / ComputeProtoFileName(2), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "26" / ComputeProtoFileName(5), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "26" / ComputeProtoFileName(8), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "26" / ComputeProtoFileName(12), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "26" / ComputeProtoFileName(15), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "26" / ComputeProtoFileName(18), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "26" / ComputeProtoFileName(22), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "27" / ComputeProtoFileName(1), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "27" / ComputeProtoFileName(4), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "27" / ComputeProtoFileName(8), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "27" / ComputeProtoFileName(11), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "27" / ComputeProtoFileName(14), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "27" / ComputeProtoFileName(18), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "27" / ComputeProtoFileName(21), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "28" / ComputeProtoFileName(0), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "28" / ComputeProtoFileName(4), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "28" / ComputeProtoFileName(7), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "28" / ComputeProtoFileName(10), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "28" / ComputeProtoFileName(14), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "28" / ComputeProtoFileName(17), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "28" / ComputeProtoFileName(20), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "28" / ComputeProtoFileName(23), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "29" / ComputeProtoFileName(3), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "29" / ComputeProtoFileName(6), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "29" / ComputeProtoFileName(9), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "29" / ComputeProtoFileName(13), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "29" / ComputeProtoFileName(16), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "29" / ComputeProtoFileName(19), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "29" / ComputeProtoFileName(23), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "30" / ComputeProtoFileName(2), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "30" / ComputeProtoFileName(5), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "30" / ComputeProtoFileName(9), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "30" / ComputeProtoFileName(12), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "30" / ComputeProtoFileName(15), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "30" / ComputeProtoFileName(19), + subPath1 / std::string_view{mk1.str()} / "2002" / "06" / "30" / ComputeProtoFileName(22), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "01" / ComputeProtoFileName(1), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "01" / ComputeProtoFileName(5), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "01" / ComputeProtoFileName(8), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "01" / ComputeProtoFileName(11), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "01" / ComputeProtoFileName(14), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "01" / ComputeProtoFileName(18), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "01" / ComputeProtoFileName(21), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "02" / ComputeProtoFileName(0), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "02" / ComputeProtoFileName(4), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "02" / ComputeProtoFileName(7), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "02" / ComputeProtoFileName(10), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "02" / ComputeProtoFileName(14), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "02" / ComputeProtoFileName(17), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "02" / ComputeProtoFileName(20), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "03" / ComputeProtoFileName(0), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "03" / ComputeProtoFileName(3), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "03" / ComputeProtoFileName(6), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "03" / ComputeProtoFileName(10), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "03" / ComputeProtoFileName(13), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "03" / ComputeProtoFileName(16), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "03" / ComputeProtoFileName(20), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "03" / ComputeProtoFileName(23), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "04" / ComputeProtoFileName(2), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "04" / ComputeProtoFileName(5), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "04" / ComputeProtoFileName(9), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "04" / ComputeProtoFileName(12), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "04" / ComputeProtoFileName(15), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "04" / ComputeProtoFileName(19), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "04" / ComputeProtoFileName(22), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "05" / ComputeProtoFileName(1), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "05" / ComputeProtoFileName(5), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "05" / ComputeProtoFileName(8), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "05" / ComputeProtoFileName(11), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "05" / ComputeProtoFileName(15), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "05" / ComputeProtoFileName(18), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "05" / ComputeProtoFileName(21), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "06" / ComputeProtoFileName(1), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "06" / ComputeProtoFileName(4), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "06" / ComputeProtoFileName(7), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "06" / ComputeProtoFileName(11), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "06" / ComputeProtoFileName(14), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "06" / ComputeProtoFileName(17), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "06" / ComputeProtoFileName(21), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "07" / ComputeProtoFileName(0), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "07" / ComputeProtoFileName(3), + subPath1 / std::string_view{mk1.str()} / "2002" / "07" / "07" / ComputeProtoFileName(6), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "25" / ComputeProtoFileName(4), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "25" / ComputeProtoFileName(8), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "25" / ComputeProtoFileName(11), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "25" / ComputeProtoFileName(14), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "25" / ComputeProtoFileName(18), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "25" / ComputeProtoFileName(21), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "26" / ComputeProtoFileName(0), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "26" / ComputeProtoFileName(3), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "26" / ComputeProtoFileName(7), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "26" / ComputeProtoFileName(10), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "26" / ComputeProtoFileName(13), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "26" / ComputeProtoFileName(17), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "26" / ComputeProtoFileName(20), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "26" / ComputeProtoFileName(23), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "27" / ComputeProtoFileName(3), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "27" / ComputeProtoFileName(6), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "27" / ComputeProtoFileName(9), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "27" / ComputeProtoFileName(13), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "27" / ComputeProtoFileName(16), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "27" / ComputeProtoFileName(19), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "27" / ComputeProtoFileName(23), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "28" / ComputeProtoFileName(2), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "28" / ComputeProtoFileName(5), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "28" / ComputeProtoFileName(9), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "28" / ComputeProtoFileName(12), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "28" / ComputeProtoFileName(15), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "28" / ComputeProtoFileName(19), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "28" / ComputeProtoFileName(22), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "29" / ComputeProtoFileName(1), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "29" / ComputeProtoFileName(4), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "29" / ComputeProtoFileName(8), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "29" / ComputeProtoFileName(11), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "29" / ComputeProtoFileName(14), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "29" / ComputeProtoFileName(18), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "29" / ComputeProtoFileName(21), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "30" / ComputeProtoFileName(0), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "30" / ComputeProtoFileName(4), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "30" / ComputeProtoFileName(7), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "30" / ComputeProtoFileName(10), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "30" / ComputeProtoFileName(14), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "30" / ComputeProtoFileName(17), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "30" / ComputeProtoFileName(20), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "31" / ComputeProtoFileName(0), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "31" / ComputeProtoFileName(3), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "31" / ComputeProtoFileName(6), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "31" / ComputeProtoFileName(10), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "31" / ComputeProtoFileName(13), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "31" / ComputeProtoFileName(16), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "31" / ComputeProtoFileName(19), + subPath1 / std::string_view{mk4.str()} / "1999" / "03" / "31" / ComputeProtoFileName(23), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "01" / ComputeProtoFileName(2), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "01" / ComputeProtoFileName(5), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "01" / ComputeProtoFileName(9), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "01" / ComputeProtoFileName(12), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "01" / ComputeProtoFileName(15), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "01" / ComputeProtoFileName(19), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "01" / ComputeProtoFileName(22), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "02" / ComputeProtoFileName(1), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "02" / ComputeProtoFileName(5), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "02" / ComputeProtoFileName(8), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "02" / ComputeProtoFileName(11), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "02" / ComputeProtoFileName(15), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "02" / ComputeProtoFileName(18), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "02" / ComputeProtoFileName(21), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "03" / ComputeProtoFileName(1), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "03" / ComputeProtoFileName(4), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "03" / ComputeProtoFileName(7), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "03" / ComputeProtoFileName(10), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "03" / ComputeProtoFileName(14), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "03" / ComputeProtoFileName(17), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "03" / ComputeProtoFileName(20), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "04" / ComputeProtoFileName(0), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "04" / ComputeProtoFileName(3), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "04" / ComputeProtoFileName(6), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "04" / ComputeProtoFileName(10), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "04" / ComputeProtoFileName(13), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "04" / ComputeProtoFileName(16), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "04" / ComputeProtoFileName(20), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "04" / ComputeProtoFileName(23), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "05" / ComputeProtoFileName(2), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "05" / ComputeProtoFileName(6), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "05" / ComputeProtoFileName(9), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "05" / ComputeProtoFileName(12), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "05" / ComputeProtoFileName(16), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "05" / ComputeProtoFileName(19), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "05" / ComputeProtoFileName(22), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "06" / ComputeProtoFileName(2), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "06" / ComputeProtoFileName(5), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "06" / ComputeProtoFileName(8), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "06" / ComputeProtoFileName(11), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "06" / ComputeProtoFileName(15), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "06" / ComputeProtoFileName(18), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "06" / ComputeProtoFileName(21), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "07" / ComputeProtoFileName(1), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "07" / ComputeProtoFileName(4), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "07" / ComputeProtoFileName(7), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "07" / ComputeProtoFileName(11), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "07" / ComputeProtoFileName(14), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "07" / ComputeProtoFileName(17), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "07" / ComputeProtoFileName(21), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "08" / ComputeProtoFileName(0), + subPath1 / std::string_view{mk4.str()} / "1999" / "04" / "08" / ComputeProtoFileName(3), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "23" / ComputeProtoFileName(7), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "23" / ComputeProtoFileName(11), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "23" / ComputeProtoFileName(14), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "23" / ComputeProtoFileName(17), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "23" / ComputeProtoFileName(21), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "24" / ComputeProtoFileName(0), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "24" / ComputeProtoFileName(3), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "24" / ComputeProtoFileName(7), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "24" / ComputeProtoFileName(10), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "24" / ComputeProtoFileName(13), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "24" / ComputeProtoFileName(17), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "24" / ComputeProtoFileName(20), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "24" / ComputeProtoFileName(23), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "25" / ComputeProtoFileName(3), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "25" / ComputeProtoFileName(6), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "25" / ComputeProtoFileName(9), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "25" / ComputeProtoFileName(13), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "25" / ComputeProtoFileName(16), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "25" / ComputeProtoFileName(19), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "25" / ComputeProtoFileName(22), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "26" / ComputeProtoFileName(2), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "26" / ComputeProtoFileName(5), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "26" / ComputeProtoFileName(8), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "26" / ComputeProtoFileName(12), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "26" / ComputeProtoFileName(15), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "26" / ComputeProtoFileName(18), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "26" / ComputeProtoFileName(22), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "27" / ComputeProtoFileName(1), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "27" / ComputeProtoFileName(4), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "27" / ComputeProtoFileName(8), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "27" / ComputeProtoFileName(11), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "27" / ComputeProtoFileName(14), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "27" / ComputeProtoFileName(18), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "27" / ComputeProtoFileName(21), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "28" / ComputeProtoFileName(0), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "28" / ComputeProtoFileName(4), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "28" / ComputeProtoFileName(7), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "28" / ComputeProtoFileName(10), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "28" / ComputeProtoFileName(14), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "28" / ComputeProtoFileName(17), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "28" / ComputeProtoFileName(20), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "28" / ComputeProtoFileName(23), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "29" / ComputeProtoFileName(3), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "29" / ComputeProtoFileName(6), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "29" / ComputeProtoFileName(9), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "29" / ComputeProtoFileName(13), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "29" / ComputeProtoFileName(16), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "29" / ComputeProtoFileName(19), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "29" / ComputeProtoFileName(23), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "30" / ComputeProtoFileName(2), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "30" / ComputeProtoFileName(5), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "30" / ComputeProtoFileName(9), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "30" / ComputeProtoFileName(12), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "30" / ComputeProtoFileName(15), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "30" / ComputeProtoFileName(19), + subPath1 / std::string_view{mk4.str()} / "2002" / "06" / "30" / ComputeProtoFileName(22), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "01" / ComputeProtoFileName(1), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "01" / ComputeProtoFileName(5), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "01" / ComputeProtoFileName(8), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "01" / ComputeProtoFileName(11), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "01" / ComputeProtoFileName(14), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "01" / ComputeProtoFileName(18), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "01" / ComputeProtoFileName(21), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "02" / ComputeProtoFileName(0), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "02" / ComputeProtoFileName(4), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "02" / ComputeProtoFileName(7), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "02" / ComputeProtoFileName(10), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "02" / ComputeProtoFileName(14), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "02" / ComputeProtoFileName(17), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "02" / ComputeProtoFileName(20), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "03" / ComputeProtoFileName(0), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "03" / ComputeProtoFileName(3), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "03" / ComputeProtoFileName(6), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "03" / ComputeProtoFileName(10), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "03" / ComputeProtoFileName(13), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "03" / ComputeProtoFileName(16), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "03" / ComputeProtoFileName(20), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "03" / ComputeProtoFileName(23), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "04" / ComputeProtoFileName(2), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "04" / ComputeProtoFileName(5), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "04" / ComputeProtoFileName(9), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "04" / ComputeProtoFileName(12), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "04" / ComputeProtoFileName(15), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "04" / ComputeProtoFileName(19), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "04" / ComputeProtoFileName(22), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "05" / ComputeProtoFileName(1), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "05" / ComputeProtoFileName(5), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "05" / ComputeProtoFileName(8), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "05" / ComputeProtoFileName(11), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "05" / ComputeProtoFileName(15), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "05" / ComputeProtoFileName(18), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "05" / ComputeProtoFileName(21), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "06" / ComputeProtoFileName(1), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "06" / ComputeProtoFileName(4), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "06" / ComputeProtoFileName(7), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "06" / ComputeProtoFileName(11), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "06" / ComputeProtoFileName(14), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "06" / ComputeProtoFileName(17), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "06" / ComputeProtoFileName(21), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "07" / ComputeProtoFileName(0), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "07" / ComputeProtoFileName(3), + subPath1 / std::string_view{mk4.str()} / "2002" / "07" / "07" / ComputeProtoFileName(6), + }; + + auto isFilePresent = [](const auto &fileName) { return std::filesystem::exists(fileName); }; + + EXPECT_TRUE(std::ranges::all_of(kExpectedFiles, isFilePresent)); + + auto deserializer = createDeserializer(); + auto marketTimestampSet = deserializer.listMarkets(timeWindowAll); + + ASSERT_EQ(marketTimestampSet.size(), 2U); + + const auto &marketTimestamp = marketTimestampSet.front(); + + const auto lastTp = *std::next(std::end(kTimePoints), -1); + + EXPECT_EQ(marketTimestamp.market, mk1); + EXPECT_GT(marketTimestamp.timePoint + kDurationStep, lastTp + kDurationRange); + + for (Market market : kMarkets) { + auto allData = deserializer.loadMarket(market, timeWindowAll); + + EXPECT_EQ(pushedPublicTrades[market], allData); + + auto partialData = + deserializer.loadMarket(market, TimeWindow{kTimePoints[0], kTimePoints[0] + std::chrono::days(1)}); + + // below check is an assert because we should not launch the std::equal below if this condition is not satisfied + ASSERT_LT(partialData.size(), allData.size()); + + EXPECT_EQ(partialData.size(), 8U); + EXPECT_TRUE( + std::equal(partialData.begin(), partialData.end(), allData.begin(), allData.begin() + partialData.size())); + } +} + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/test/proto-test-data.hpp b/src/serialization/test/proto-test-data.hpp new file mode 100644 index 00000000..5ef1eea7 --- /dev/null +++ b/src/serialization/test/proto-test-data.hpp @@ -0,0 +1,69 @@ +#pragma once + +#include + +#include +#include + +#include "market.hpp" +#include "monetaryamount.hpp" +#include "proto-public-trade-converter.hpp" +#include "public-trade.pb.h" +#include "publictrade.hpp" +#include "time-window.hpp" +#include "timedef.hpp" +#include "tradeside.hpp" + +namespace cct { + +class ProtobufBaseDataTest : public ::testing::Test { + protected: + TimePoint tp1{milliseconds{std::numeric_limits::max() / 10000000}}; + TimePoint tp2{milliseconds{std::numeric_limits::max() / 9000000}}; + TimePoint tp3{milliseconds{std::numeric_limits::max() / 8000000}}; + TimePoint tp4{milliseconds{std::numeric_limits::max() / 7000000}}; + TimePoint tp5{milliseconds{std::numeric_limits::max() / 6900000}}; + TimePoint tp6{milliseconds{std::numeric_limits::max() / 6800000}}; + TimePoint tp7{milliseconds{std::numeric_limits::max() / 6700000}}; + TimePoint tp8{milliseconds{std::numeric_limits::max() / 6600000}}; + TimePoint tp9{milliseconds{std::numeric_limits::max() / 6500000}}; + TimePoint tp10 = tp9 + milliseconds{1}; + + TimeWindow timeWindowAll{tp1, tp10 + milliseconds{1}}; + TimeWindow timeWindow14{tp1, tp5}; + TimeWindow timeWindow79{tp7, tp9 + milliseconds{1}}; + + Market mk1{"ETH", "USDT"}; + Market mk2{"BTC", "USD"}; + Market mk3{"SHIB", "USDT"}; + Market mk4{"SOL", "BTC"}; + Market mk5{"SOL", "ETH"}; + Market mk6{"ETH", "BTC"}; + Market mk7{"DOGE", "CAD"}; + + PublicTrade pt1{TradeSide::kBuy, MonetaryAmount{"0.13", mk1.base()}, MonetaryAmount{"1500.5", mk1.quote()}, tp1}; + PublicTrade pt2{TradeSide::kSell, MonetaryAmount{"3.7", mk1.base()}, MonetaryAmount{"1500.5", mk1.quote()}, tp2}; + PublicTrade pt3{TradeSide::kBuy, MonetaryAmount{"0.004", mk1.base()}, MonetaryAmount{1501, mk1.quote()}, tp3}; + PublicTrade pt4{TradeSide::kBuy, MonetaryAmount{"44473434", mk3.base()}, MonetaryAmount{"0.00045", mk3.quote()}, tp1}; + PublicTrade pt5{TradeSide::kBuy, MonetaryAmount{"45.0986", mk4.base()}, MonetaryAmount{"0.00045", mk4.quote()}, tp7}; + PublicTrade pt6{TradeSide::kSell, MonetaryAmount{"0.81153", mk6.base()}, MonetaryAmount{"0.0834", mk6.quote()}, tp6}; + PublicTrade pt7{TradeSide::kSell, MonetaryAmount{694873, mk7.base()}, MonetaryAmount{"0.045", mk7.quote()}, tp8}; + PublicTrade pt8{TradeSide::kSell, MonetaryAmount{"0.1", mk2.base()}, MonetaryAmount{50000, mk2.quote()}, tp4}; + PublicTrade pt9{TradeSide::kSell, MonetaryAmount{"56", mk1.base()}, MonetaryAmount{1300, mk1.quote()}, tp5}; + PublicTrade pt10{TradeSide::kBuy, MonetaryAmount{"37.8", mk5.base()}, MonetaryAmount{"0.032", mk5.quote()}, tp9}; + PublicTrade pt11{TradeSide::kBuy, MonetaryAmount{"12.2", mk5.base()}, MonetaryAmount{"0.033", mk5.quote()}, tp10}; + + ::proto::PublicTrade td1{ConvertPublicTradeToProto(pt1)}; + ::proto::PublicTrade td2{ConvertPublicTradeToProto(pt2)}; + ::proto::PublicTrade td3{ConvertPublicTradeToProto(pt3)}; + ::proto::PublicTrade td4{ConvertPublicTradeToProto(pt4)}; + ::proto::PublicTrade td5{ConvertPublicTradeToProto(pt5)}; + ::proto::PublicTrade td6{ConvertPublicTradeToProto(pt6)}; + ::proto::PublicTrade td7{ConvertPublicTradeToProto(pt7)}; + ::proto::PublicTrade td8{ConvertPublicTradeToProto(pt8)}; + ::proto::PublicTrade td9{ConvertPublicTradeToProto(pt9)}; + ::proto::PublicTrade td10{ConvertPublicTradeToProto(pt10)}; + ::proto::PublicTrade td11{ConvertPublicTradeToProto(pt11)}; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/serialization/test/serialization-tools_test.cpp b/src/serialization/test/serialization-tools_test.cpp new file mode 100644 index 00000000..5da6a1c4 --- /dev/null +++ b/src/serialization/test/serialization-tools_test.cpp @@ -0,0 +1,40 @@ +#include "serialization-tools.hpp" + +#include + +namespace cct { + +TEST(SerializationTools, ComputeProtoSubPath) { + EXPECT_EQ(ComputeProtoSubPath("/path/to/data", "upbit", "order-books"), "/path/to/data/serialized/order-books/upbit"); + EXPECT_EQ(ComputeProtoSubPath("another-path", "bithumb", "trades"), "another-path/serialized/trades/bithumb"); + EXPECT_EQ(ComputeProtoSubPath(".", "kraken", "trades"), "./serialized/trades/kraken"); + EXPECT_EQ(ComputeProtoSubPath("/path/to/data", "binance", "order-books"), + "/path/to/data/serialized/order-books/binance"); + EXPECT_EQ(ComputeProtoSubPath("/path/to/data", "kucoin", "trades"), "/path/to/data/serialized/trades/kucoin"); +} + +TEST(SerializationTools, MonthStr) { + EXPECT_EQ(MonthStr(1), "01"); + EXPECT_EQ(MonthStr(6), "06"); + EXPECT_EQ(MonthStr(10), "10"); + EXPECT_EQ(MonthStr(11), "11"); + EXPECT_EQ(MonthStr(12), "12"); +} + +TEST(SerializationTools, DayOfMonthStr) { + EXPECT_EQ(DayOfMonthStr(1), "01"); + EXPECT_EQ(DayOfMonthStr(2), "02"); + EXPECT_EQ(DayOfMonthStr(10), "10"); + EXPECT_EQ(DayOfMonthStr(17), "17"); + EXPECT_EQ(DayOfMonthStr(22), "22"); + EXPECT_EQ(DayOfMonthStr(31), "31"); +} + +TEST(SerializationTools, ComputeProtoFileName) { + EXPECT_EQ(ComputeProtoFileName(0), "00-00-00_00-59-59.binpb"); + EXPECT_EQ(ComputeProtoFileName(4), "04-00-00_04-59-59.binpb"); + EXPECT_EQ(ComputeProtoFileName(17), "17-00-00_17-59-59.binpb"); + EXPECT_EQ(ComputeProtoFileName(23), "23-00-00_23-59-59.binpb"); +} + +} // namespace cct \ No newline at end of file diff --git a/src/tech/include/durationstring.hpp b/src/tech/include/durationstring.hpp index 0eccb923..1cab4a6a 100644 --- a/src/tech/include/durationstring.hpp +++ b/src/tech/include/durationstring.hpp @@ -7,6 +7,10 @@ namespace cct { +/// Check if 'str' starts with a Duration. +/// Returns the duration string length (0 if no duration detected) +std::string_view::size_type DurationLen(std::string_view str); + /// Parse given string representation of a duration and return the duration. /// Amounts and units may be separated by spaces. For example: /// "1h45min" is allowed, as well as "1h 45min" and "1 h 45 min " diff --git a/src/tech/include/unitsparser.hpp b/src/tech/include/unitsparser.hpp index 9a2a95ed..e7cfd975 100644 --- a/src/tech/include/unitsparser.hpp +++ b/src/tech/include/unitsparser.hpp @@ -4,6 +4,7 @@ #include namespace cct { + /// Parses a string representation of a number of bytes. /// string should contain an integral number (decimal not supported) possibly followed by one of these units: /// - T, G, M, k for multiples of 1000 @@ -11,4 +12,5 @@ namespace cct { /// Note: it is a simplified version of the syntax used by Kubernetes: /// https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ int64_t ParseNumberOfBytes(std::string_view sizeStr); + } // namespace cct \ No newline at end of file diff --git a/src/tech/src/durationstring.cpp b/src/tech/src/durationstring.cpp index 77ad4ffa..6cd6f5f2 100644 --- a/src/tech/src/durationstring.cpp +++ b/src/tech/src/durationstring.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -23,6 +24,41 @@ constexpr std::pair kDurationUnits[] = { }; } +std::string_view::size_type DurationLen(std::string_view str) { + const auto sz = str.size(); + if (sz == 0) { + return 0; + } + std::string_view::size_type charPos{}; + while (charPos < sz && isspace(str[charPos])) { + ++charPos; + } + int value{}; + const auto [ptr, err] = std::from_chars(str.data() + charPos, str.data() + str.size(), value); + if (err != std::errc() || value <= 0) { + return 0; + } + charPos = ptr - str.data(); + + while (charPos < sz && isspace(str[charPos])) { + ++charPos; + } + const auto first = charPos; + while (charPos < sz && islower(str[charPos])) { + ++charPos; + } + const std::string_view timeUnitStr(str.begin() + first, str.begin() + charPos); + + const auto it = std::ranges::find_if(kDurationUnits, [timeUnitStr](const auto &durationUnitWithDuration) { + return durationUnitWithDuration.first == timeUnitStr; + }); + if (it == std::end(kDurationUnits)) { + return 0; + } + // There is a substring with size 'charPos' that represents a duration + return charPos + DurationLen(str.substr(charPos)); +} + Duration ParseDuration(std::string_view durationStr) { while (!durationStr.empty() && isspace(durationStr.front())) { durationStr.remove_prefix(1); diff --git a/src/tech/test/durationstring_test.cpp b/src/tech/test/durationstring_test.cpp index 96cb74ae..a524f133 100644 --- a/src/tech/test/durationstring_test.cpp +++ b/src/tech/test/durationstring_test.cpp @@ -9,6 +9,20 @@ namespace cct { +TEST(DurationLen, Basic) { EXPECT_EQ(DurationLen("99min"), 5); } + +TEST(DurationLen, BasicComplex) { EXPECT_EQ(DurationLen("34d45min"), 8); } + +TEST(DurationLen, BasicWithComma) { EXPECT_EQ(DurationLen("23s,bithumb"), 3); } + +TEST(DurationLen, ComplexWithSpaces) { EXPECT_EQ(DurationLen(" 1 d 52 h,kraken"), 9); } + +TEST(DurationLen, NegativeValue) { EXPECT_EQ(DurationLen("-3sec"), 0); } + +TEST(DurationLen, InvalidTimeUnit) { EXPECT_EQ(DurationLen("63po"), 0); } + +TEST(DurationLen, DoesNotStartWithNumber) { EXPECT_EQ(DurationLen("us"), 0); } + TEST(ParseDuration, EmptyDurationNotAllowed) { EXPECT_THROW(ParseDuration(""), invalid_argument); } TEST(ParseDuration, DurationDays) { EXPECT_EQ(ParseDuration("37d"), std::chrono::days(37)); } diff --git a/src/trading/CMakeLists.txt b/src/trading/CMakeLists.txt new file mode 100644 index 00000000..5df5ce59 --- /dev/null +++ b/src/trading/CMakeLists.txt @@ -0,0 +1,2 @@ +add_subdirectory(algorithms) +add_subdirectory(common) \ No newline at end of file diff --git a/src/trading/algorithms/CMakeLists.txt b/src/trading/algorithms/CMakeLists.txt new file mode 100644 index 00000000..51e64005 --- /dev/null +++ b/src/trading/algorithms/CMakeLists.txt @@ -0,0 +1,10 @@ +aux_source_directory(src TRADING-ALGORITHMS_SRC) + +add_library(coincenter_trading-algorithms STATIC ${TRADING-ALGORITHMS_SRC}) + +target_link_libraries(coincenter_trading-algorithms PUBLIC coincenter_api-objects) +target_link_libraries(coincenter_trading-algorithms PUBLIC coincenter_objects) +target_link_libraries(coincenter_trading-algorithms PUBLIC coincenter_tech) +target_link_libraries(coincenter_trading-algorithms PUBLIC coincenter_trading-common) + +target_include_directories(coincenter_trading-algorithms PUBLIC include) \ No newline at end of file diff --git a/src/trading/algorithms/include/dummy-market-trader.hpp b/src/trading/algorithms/include/dummy-market-trader.hpp new file mode 100644 index 00000000..6d64c817 --- /dev/null +++ b/src/trading/algorithms/include/dummy-market-trader.hpp @@ -0,0 +1,20 @@ +#pragma once + +#include "abstract-market-trader.hpp" +#include "trader-command.hpp" + +namespace cct { + +class MarketDataView; +class MarketTraderEngineState; + +class DummyMarketTrader : public AbstractMarketTrader { + public: + static constexpr std::string_view kName = "dummy-trader"; + + DummyMarketTrader(const MarketTraderEngineState &marketTraderEngineState) noexcept; + + TraderCommand trade([[maybe_unused]] const MarketDataView &marketDataView) override; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/trading/algorithms/include/example-market-trader.hpp b/src/trading/algorithms/include/example-market-trader.hpp new file mode 100644 index 00000000..90917d4a --- /dev/null +++ b/src/trading/algorithms/include/example-market-trader.hpp @@ -0,0 +1,20 @@ +#pragma once + +#include "abstract-market-trader.hpp" +#include "trader-command.hpp" + +namespace cct { + +class MarketDataView; +class MarketTraderEngineState; + +class ExampleMarketTrader : public AbstractMarketTrader { + public: + static constexpr std::string_view kName = "example-trader"; + + ExampleMarketTrader(const MarketTraderEngineState &marketTraderEngineState) noexcept; + + TraderCommand trade([[maybe_unused]] const MarketDataView &marketDataView) override; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/trading/algorithms/include/market-trader-factory.hpp b/src/trading/algorithms/include/market-trader-factory.hpp new file mode 100644 index 00000000..a4cfb3d6 --- /dev/null +++ b/src/trading/algorithms/include/market-trader-factory.hpp @@ -0,0 +1,22 @@ +#pragma once + +#include +#include + +#include "abstract-market-trader-factory.hpp" +#include "abstract-market-trader.hpp" + +namespace cct { + +class MarketTraderEngineState; + +class MarketTraderFactory : public AbstractMarketTraderFactory { + public: + std::span allSupportedAlgorithms() const override; + + /// Creates a new MarketTrader from the underlying type of the algorithm name. + /// For instance, create("dummy-trader") will return a DummyMarketTrader. + std::unique_ptr construct( + std::string_view algorithmName, const MarketTraderEngineState& marketTraderEngineState) const override; +}; +} // namespace cct \ No newline at end of file diff --git a/src/trading/algorithms/src/dummy-market-trader.cpp b/src/trading/algorithms/src/dummy-market-trader.cpp new file mode 100644 index 00000000..2b034b03 --- /dev/null +++ b/src/trading/algorithms/src/dummy-market-trader.cpp @@ -0,0 +1,17 @@ +#include "dummy-market-trader.hpp" + +#include "abstract-market-trader.hpp" +#include "market-data-view.hpp" +#include "market-trader-engine-state.hpp" +#include "trader-command.hpp" + +namespace cct { + +DummyMarketTrader::DummyMarketTrader(const MarketTraderEngineState &marketTraderEngineState) noexcept + : AbstractMarketTrader(kName, marketTraderEngineState) {} + +TraderCommand DummyMarketTrader::trade([[maybe_unused]] const MarketDataView &marketDataView) { + return TraderCommand::Wait(); +} + +} // namespace cct \ No newline at end of file diff --git a/src/trading/algorithms/src/example-market-trader.cpp b/src/trading/algorithms/src/example-market-trader.cpp new file mode 100644 index 00000000..f763859b --- /dev/null +++ b/src/trading/algorithms/src/example-market-trader.cpp @@ -0,0 +1,17 @@ +#include "example-market-trader.hpp" + +#include "abstract-market-trader.hpp" +#include "market-data-view.hpp" +#include "market-trader-engine-state.hpp" +#include "trader-command.hpp" + +namespace cct { + +ExampleMarketTrader::ExampleMarketTrader(const MarketTraderEngineState &marketTraderEngineState) noexcept + : AbstractMarketTrader(kName, marketTraderEngineState) {} + +TraderCommand ExampleMarketTrader::trade([[maybe_unused]] const MarketDataView &marketDataView) { + return TraderCommand::Place(TradeSide::kSell); +} + +} // namespace cct \ No newline at end of file diff --git a/src/trading/algorithms/src/market-trader-factory.cpp b/src/trading/algorithms/src/market-trader-factory.cpp new file mode 100644 index 00000000..a6a01138 --- /dev/null +++ b/src/trading/algorithms/src/market-trader-factory.cpp @@ -0,0 +1,33 @@ +#include "market-trader-factory.hpp" + +#include +#include +#include + +#include "cct_invalid_argument_exception.hpp" +#include "dummy-market-trader.hpp" +#include "example-market-trader.hpp" + +namespace cct { + +class MarketTraderEngineState; + +std::span MarketTraderFactory::allSupportedAlgorithms() const { + static constexpr std::string_view kAllAlgorithms[] = {DummyMarketTrader::kName, ExampleMarketTrader::kName}; + return kAllAlgorithms; +} + +std::unique_ptr MarketTraderFactory::construct( + std::string_view algorithmName, const MarketTraderEngineState &marketTraderEngineState) const { + if (algorithmName == DummyMarketTrader::kName) { + return std::make_unique(marketTraderEngineState); + } + + if (algorithmName == ExampleMarketTrader::kName) { + return std::make_unique(marketTraderEngineState); + } + + throw invalid_argument("Unknown trader algorithm '{}'", algorithmName); +} + +} // namespace cct \ No newline at end of file diff --git a/src/trading/common/CMakeLists.txt b/src/trading/common/CMakeLists.txt new file mode 100644 index 00000000..cd1bb22f --- /dev/null +++ b/src/trading/common/CMakeLists.txt @@ -0,0 +1,7 @@ +aux_source_directory(src TRADING-COMMON_SRC) + +add_library(coincenter_trading-common STATIC ${TRADING-COMMON_SRC}) +target_link_libraries(coincenter_trading-common PUBLIC coincenter_api-objects) +target_link_libraries(coincenter_trading-common PUBLIC coincenter_objects) +target_link_libraries(coincenter_trading-common PUBLIC coincenter_tech) +target_include_directories(coincenter_trading-common PUBLIC include) \ No newline at end of file diff --git a/src/trading/common/include/abstract-market-trader-factory.hpp b/src/trading/common/include/abstract-market-trader-factory.hpp new file mode 100644 index 00000000..14bbbaa8 --- /dev/null +++ b/src/trading/common/include/abstract-market-trader-factory.hpp @@ -0,0 +1,24 @@ +#pragma once + +#include +#include +#include + +namespace cct { + +class AbstractMarketTrader; +class MarketTraderEngineState; + +/// Interface that you need to derive to provide your own algorithms to coincenter. +class AbstractMarketTraderFactory { + public: + /// Returns a span of all supported algorithms of this market trader factory. + virtual std::span allSupportedAlgorithms() const = 0; + + /// Creates a new MarketTrader from the underlying type of the algorithm name. + /// For instance, create("dummy-trader") will return a DummyMarketTrader. + virtual std::unique_ptr construct( + std::string_view algorithmName, const MarketTraderEngineState& marketTraderEngineState) const = 0; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/trading/common/include/abstract-market-trader.hpp b/src/trading/common/include/abstract-market-trader.hpp new file mode 100644 index 00000000..db7a41ae --- /dev/null +++ b/src/trading/common/include/abstract-market-trader.hpp @@ -0,0 +1,36 @@ +#pragma once + +#include + +#include "trader-command.hpp" + +namespace cct { + +class MarketDataView; +class MarketTraderEngineState; + +/// Base class for a trading algorithm. +/// It can be derived and only need to implement the trade method to be used in the MarketTraderEngine. +/// the Market Trader Engine state is also provided as a const reference to have the data of the context (orders, +/// available amounts). +class AbstractMarketTrader { + public: + virtual ~AbstractMarketTrader() = default; + + virtual TraderCommand trade(const MarketDataView &marketDataView) = 0; + + std::string_view name() const { return _name; } + + const MarketTraderEngineState &marketTraderEngineState() const { return _marketTraderEngineState; } + + protected: + /// Constructs a new AbstractMarketTrader. + /// @param name should be a view to a constant string as only a std::string_view will be stored in this object. + AbstractMarketTrader(std::string_view name, const MarketTraderEngineState &marketTraderEngineState) noexcept; + + private: + std::string_view _name; + const MarketTraderEngineState &_marketTraderEngineState; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/trading/common/include/market-data-view.hpp b/src/trading/common/include/market-data-view.hpp new file mode 100644 index 00000000..8e08b83a --- /dev/null +++ b/src/trading/common/include/market-data-view.hpp @@ -0,0 +1,46 @@ +#pragma once + +#include +#include + +#include "marketorderbook.hpp" +#include "publictrade.hpp" +#include "timedef.hpp" + +namespace cct { + +/// A class providing a view to current and historical market data for the market trader. +class MarketDataView { + public: + /// Get a reference to last (current for this turn) market order book + const MarketOrderBook ¤tMarketOrderBook() const { return _pOrderBooks[_currentOrderBookEndPos - 1U]; } + + /// Get a span of all historical market order books since the start of the market trader engine (including current / + /// last one) + std::span pastMarketOrderBooks() const { return {_pOrderBooks, _currentOrderBookEndPos}; } + + /// Get a span of all new public trades that occurred before last (current for this turn) market order book that have + /// not been seen before. + std::span currentPublicTrades() const { return {_pCurrentTradesBeg, _pCurrentTradesEnd}; } + + /// Get a span of all public trades since the start of the market trader engine (including current / last ones). + std::span pastPublicTrades() const { return {_pPublicTradesBeg, _pCurrentTradesEnd}; } + + private: + friend class MarketTraderEngine; + + MarketDataView(const MarketOrderBook *pOrderBooks, const PublicTrade *pPublicTradesBeg, + const PublicTrade *pPublicTradesEnd) noexcept; + + void advanceUntil(TimePoint marketOrderBookTs); + + const MarketOrderBook *_pOrderBooks; + const PublicTrade *_pPublicTradesBeg; + const PublicTrade *_pPublicTradesEnd; + + const PublicTrade *_pCurrentTradesBeg; + const PublicTrade *_pCurrentTradesEnd; + std::size_t _currentOrderBookEndPos{}; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/trading/common/include/market-trader-engine-state.hpp b/src/trading/common/include/market-trader-engine-state.hpp new file mode 100644 index 00000000..a2951519 --- /dev/null +++ b/src/trading/common/include/market-trader-engine-state.hpp @@ -0,0 +1,69 @@ +#pragma once + +#include +#include +#include + +#include "cct_type_traits.hpp" +#include "closed-order.hpp" +#include "exchangeconfig.hpp" +#include "exchangeprivateapitypes.hpp" +#include "monetaryamount.hpp" +#include "opened-order.hpp" +#include "stringhelpers.hpp" +#include "trader-command.hpp" + +namespace cct { + +/// Contains the mutable state of the market trader engine. +class MarketTraderEngineState { + public: + MarketTraderEngineState(MonetaryAmount startAmountBase, MonetaryAmount startAmountQuote); + + MonetaryAmount availableBaseAmount() const { return _availableBaseAmount; } + MonetaryAmount availableQuoteAmount() const { return _availableQuoteAmount; } + + std::span openedOrders() const { return _openedOrders; } + std::span closedOrders() const { return _closedOrders; } + + using trivially_relocatable = std::bool_constant && + is_trivially_relocatable_v>::type; + + private: + friend class MarketTraderEngine; + + MonetaryAmount computeBuyFrom(TraderCommand traderCommand) const; + + MonetaryAmount computeSellVolume(TraderCommand traderCommand) const; + + void placeBuyOrder(const ExchangeConfig &exchangeConfig, TimePoint placedTime, MonetaryAmount remainingVolume, + MonetaryAmount price, MonetaryAmount matchedVolume, MonetaryAmount from, + ExchangeConfig::FeeType feeType); + + void placeSellOrder(const ExchangeConfig &exchangeConfig, TimePoint placedTime, MonetaryAmount remainingVolume, + MonetaryAmount price, MonetaryAmount matchedVolume, ExchangeConfig::FeeType feeType); + + auto nextOrderId() { return ToString(++_nextOrderId); } + + void adjustOpenedOrderRemainingVolume(const OpenedOrder &matchedOrder, MonetaryAmount newMatchedVolume); + + void countMatchedPart(const ExchangeConfig &exchangeConfig, const OpenedOrder &matchedOrder, MonetaryAmount price, + MonetaryAmount newMatchedVolume, TimePoint matchedTime); + + void cancelOpenedOrder(int32_t orderId); + + OpenedOrderVector::const_iterator findOpenedOrder(int32_t orderId); + + void cancelAllOpenedOrders(); + + void eraseClosedOpenedOrders(std::span closedOpenedOrders); + + void adjustAvailableAmountsCancel(const OpenedOrder &openedOrder); + + MonetaryAmount _availableBaseAmount; + MonetaryAmount _availableQuoteAmount; + OpenedOrderVector _openedOrders; + ClosedOrderVector _closedOrders; + int _nextOrderId{}; +}; +} // namespace cct \ No newline at end of file diff --git a/src/trading/common/include/market-trader-engine.hpp b/src/trading/common/include/market-trader-engine.hpp new file mode 100644 index 00000000..9fb686e7 --- /dev/null +++ b/src/trading/common/include/market-trader-engine.hpp @@ -0,0 +1,64 @@ +#pragma once + +#include +#include + +#include "abstract-market-trader.hpp" +#include "cct_type_traits.hpp" +#include "exchangeconfig.hpp" +#include "exchangeprivateapitypes.hpp" +#include "market-order-book-vector.hpp" +#include "market-trader-engine-state.hpp" +#include "market-trading-result.hpp" +#include "market.hpp" +#include "marketorderbook.hpp" +#include "monetaryamount.hpp" +#include "public-trade-vector.hpp" +#include "trade-range-stats.hpp" +#include "trader-command.hpp" + +namespace cct { + +class MarketTraderEngine { + public: + MarketTraderEngine(const ExchangeConfig &exchangeConfig, Market market, MonetaryAmount startAmountBase, + MonetaryAmount startAmountQuote); + + Market market() const { return {_startAmountBase.currencyCode(), _startAmountQuote.currencyCode()}; } + + void registerMarketTrader(std::unique_ptr marketTrader); + + TradeRangeStats validateRange(MarketOrderBookVector &marketOrderBooks, PublicTradeVector &publicTrades); + + TradeRangeStats validateRange(MarketOrderBookVector &&marketOrderBooks, PublicTradeVector &&publicTrades); + + TradeRangeStats tradeRange(MarketOrderBookVector &&marketOrderBooks, PublicTradeVector &&publicTrades); + + const MarketTraderEngineState &marketTraderEngineState() const { return _marketTraderEngineState; } + + MarketTradingResult finalizeAndComputeResult(); + + using trivially_relocatable = + std::bool_constant && is_trivially_relocatable_v && + is_trivially_relocatable_v>::type; + + private: + void buy(const MarketOrderBook &marketOrderBook, MonetaryAmount from, PriceStrategy priceStrategy); + void sell(const MarketOrderBook &marketOrderBook, MonetaryAmount volume, PriceStrategy priceStrategy); + + void updatePrice(const MarketOrderBook &marketOrderBook, TraderCommand traderCommand); + + void cancelCommand(int32_t orderId); + + void checkOpenedOrdersMatching(const MarketOrderBook &marketOrderBook); + + MonetaryAmount _startAmountBase; + MonetaryAmount _startAmountQuote; + const ExchangeConfig &_exchangeConfig; + std::unique_ptr _marketTrader; + Market _market; + MarketTraderEngineState _marketTraderEngineState; + OpenedOrderVector _newlyClosedOrders; + MarketOrderBook _lastMarketOrderBook; +}; +} // namespace cct \ No newline at end of file diff --git a/src/trading/common/include/market-trading-global-result.hpp b/src/trading/common/include/market-trading-global-result.hpp new file mode 100644 index 00000000..8d404460 --- /dev/null +++ b/src/trading/common/include/market-trading-global-result.hpp @@ -0,0 +1,13 @@ +#pragma once + +#include "market-trading-result.hpp" +#include "trade-range-stats.hpp" + +namespace cct { + +struct MarketTradingGlobalResult { + MarketTradingResult result; + TradeRangeStats stats; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/trading/common/include/market-trading-result.hpp b/src/trading/common/include/market-trading-result.hpp new file mode 100644 index 00000000..7c462393 --- /dev/null +++ b/src/trading/common/include/market-trading-result.hpp @@ -0,0 +1,43 @@ +#pragma once + +#include +#include + +#include "cct_type_traits.hpp" +#include "closed-order.hpp" +#include "exchangeprivateapitypes.hpp" +#include "market.hpp" +#include "monetaryamount.hpp" + +namespace cct { + +class MarketTradingResult { + public: + MarketTradingResult() noexcept = default; + + MarketTradingResult(std::string_view algorithmName, MonetaryAmount startBaseAmount, MonetaryAmount startQuoteAmount, + MonetaryAmount quoteAmountDelta, ClosedOrderVector matchedOrders); + + std::string_view algorithmName() const { return _algorithmName; } + + Market market() const { return {_startBaseAmount.currencyCode(), _startQuoteAmount.currencyCode()}; } + + MonetaryAmount startBaseAmount() const { return _startBaseAmount; } + + MonetaryAmount startQuoteAmount() const { return _startQuoteAmount; } + + MonetaryAmount quoteAmountDelta() const { return _quoteAmountDelta; } + + std::span matchedOrders() const { return _matchedOrders; } + + using trivially_relocatable = is_trivially_relocatable::type; + + private: + std::string_view _algorithmName; + MonetaryAmount _startBaseAmount; + MonetaryAmount _startQuoteAmount; + MonetaryAmount _quoteAmountDelta; + ClosedOrderVector _matchedOrders; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/trading/common/include/trade-range-stats.hpp b/src/trading/common/include/trade-range-stats.hpp new file mode 100644 index 00000000..a5499d54 --- /dev/null +++ b/src/trading/common/include/trade-range-stats.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include + +namespace cct { + +struct TradeRangeResultsStats { + int32_t nbSuccessful{}; + int32_t nbError{}; + + TradeRangeResultsStats operator+(const TradeRangeResultsStats &rhs) const { + return TradeRangeResultsStats{nbSuccessful + rhs.nbSuccessful, nbError + rhs.nbError}; + } + + TradeRangeResultsStats &operator+=(const TradeRangeResultsStats &rhs) { return *this = *this + rhs; } +}; + +struct TradeRangeStats { + TradeRangeResultsStats marketOrderBookStats; + TradeRangeResultsStats publicTradeStats; + + TradeRangeStats operator+(const TradeRangeStats &rhs) const { + return TradeRangeStats{marketOrderBookStats + rhs.marketOrderBookStats, publicTradeStats + rhs.publicTradeStats}; + } + + TradeRangeStats &operator+=(const TradeRangeStats &rhs) { return *this = *this + rhs; } +}; + +} // namespace cct \ No newline at end of file diff --git a/src/trading/common/include/trader-command.hpp b/src/trading/common/include/trader-command.hpp new file mode 100644 index 00000000..ced95e0d --- /dev/null +++ b/src/trading/common/include/trader-command.hpp @@ -0,0 +1,56 @@ +#pragma once + +#include +#include + +#include "orderid.hpp" +#include "priceoptionsdef.hpp" +#include "tradeside.hpp" + +namespace cct { + +class TraderCommand { + public: + enum class Type : int8_t { kWait, kBuy, kSell, kUpdatePrice, kCancel }; + + static constexpr int32_t kAllOrdersId = 0; + + /// Creates a wait command. + static TraderCommand Wait(); + + /// Creates a Place command with given intensity, side and strategy. It should be in the range [0, 100]. + static TraderCommand Place(TradeSide tradeSide, int8_t amountIntensityPercentage = 100, + PriceStrategy priceStrategy = PriceStrategy::kMaker); + + /// Creates a Cancel command with optional orderId. + /// If orderId is not specified (or empty string), will cancel all opened orders. + static TraderCommand Cancel(OrderIdView orderId = std::string_view()); + + /// Creates an Update command for specified orderId. + /// Equivalent to a Cancel and a Place at new price for remaining unmatched amount at the same turn. + static TraderCommand UpdatePrice(OrderIdView orderId, PriceStrategy priceStrategy = PriceStrategy::kMaker); + + int32_t orderId() const { return _orderId; } + + /// If this is a Place command, return the amount intensity percentage in [0, 100] + int8_t amountIntensityPercentage() const { return _amountIntensityPercentage; } + + Type type() const { return _type; } + + TradeSide tradeSide() const; + + PriceStrategy priceStrategy() const { return _priceStrategy; } + + private: + static constexpr int8_t kWaitValue = 0; + static constexpr int8_t kCancelValue = std::numeric_limits::min(); + + TraderCommand(Type type, int32_t orderId, int8_t amountIntensityPercentage, PriceStrategy priceStrategy); + + int32_t _orderId; + Type _type; + int8_t _amountIntensityPercentage; + PriceStrategy _priceStrategy; +}; + +} // namespace cct \ No newline at end of file diff --git a/src/trading/common/src/abstract-market-trader.cpp b/src/trading/common/src/abstract-market-trader.cpp new file mode 100644 index 00000000..3bd381c1 --- /dev/null +++ b/src/trading/common/src/abstract-market-trader.cpp @@ -0,0 +1,13 @@ +#include "abstract-market-trader.hpp" + +#include + +#include "market-trader-engine-state.hpp" + +namespace cct { + +AbstractMarketTrader::AbstractMarketTrader(std::string_view name, + const MarketTraderEngineState& marketTraderEngineState) noexcept + : _name(name), _marketTraderEngineState(marketTraderEngineState) {} + +} // namespace cct \ No newline at end of file diff --git a/src/trading/common/src/market-data-view.cpp b/src/trading/common/src/market-data-view.cpp new file mode 100644 index 00000000..14d244a2 --- /dev/null +++ b/src/trading/common/src/market-data-view.cpp @@ -0,0 +1,27 @@ +#include "market-data-view.hpp" + +#include + +#include "publictrade.hpp" + +namespace cct { + +MarketDataView::MarketDataView(const MarketOrderBook *pOrderBooks, const PublicTrade *pPublicTradesBeg, + const PublicTrade *pPublicTradesEnd) noexcept + : _pOrderBooks(pOrderBooks), + _pPublicTradesBeg(pPublicTradesBeg), + _pPublicTradesEnd(pPublicTradesEnd), + _pCurrentTradesBeg(pPublicTradesBeg), + _pCurrentTradesEnd(pPublicTradesEnd) {} + +void MarketDataView::advanceUntil(TimePoint marketOrderBookTs) { + // Advance the public trades iterator until we reach one that occurred after our current market order book + _pCurrentTradesBeg = _pCurrentTradesEnd; + _pCurrentTradesEnd = std::partition_point( + _pCurrentTradesBeg, _pPublicTradesEnd, + [marketOrderBookTs](const auto &publicTrade) { return publicTrade.time() < marketOrderBookTs; }); + + ++_currentOrderBookEndPos; +} + +} // namespace cct \ No newline at end of file diff --git a/src/trading/common/src/market-trader-engine-state.cpp b/src/trading/common/src/market-trader-engine-state.cpp new file mode 100644 index 00000000..45cb2fbe --- /dev/null +++ b/src/trading/common/src/market-trader-engine-state.cpp @@ -0,0 +1,136 @@ +#include "market-trader-engine-state.hpp" + +#include +#include + +#include "cct_exception.hpp" +#include "closed-order.hpp" +#include "exchangeconfig.hpp" +#include "monetaryamount.hpp" +#include "opened-order.hpp" +#include "timedef.hpp" +#include "trader-command.hpp" +#include "tradeside.hpp" + +namespace cct { +MarketTraderEngineState::MarketTraderEngineState(MonetaryAmount startAmountBase, MonetaryAmount startAmountQuote) + : _availableBaseAmount(startAmountBase), _availableQuoteAmount(startAmountQuote) {} + +MonetaryAmount MarketTraderEngineState::computeBuyFrom(TraderCommand traderCommand) const { + return (_availableQuoteAmount * traderCommand.amountIntensityPercentage()) / 100; +} + +MonetaryAmount MarketTraderEngineState::computeSellVolume(TraderCommand traderCommand) const { + return (_availableBaseAmount * traderCommand.amountIntensityPercentage()) / 100; +} + +void MarketTraderEngineState::placeBuyOrder(const ExchangeConfig &exchangeConfig, TimePoint placedTime, + MonetaryAmount remainingVolume, MonetaryAmount price, + MonetaryAmount matchedVolume, MonetaryAmount from, + ExchangeConfig::FeeType feeType) { + _availableBaseAmount += exchangeConfig.applyFee(matchedVolume, feeType); + _availableQuoteAmount -= from; + + if (remainingVolume == 0) { + _closedOrders.emplace_back(nextOrderId(), matchedVolume, price, placedTime, placedTime, TradeSide::kBuy); + } else { + _openedOrders.emplace_back(nextOrderId(), matchedVolume, remainingVolume, price, placedTime, TradeSide::kBuy); + } +} + +void MarketTraderEngineState::placeSellOrder(const ExchangeConfig &exchangeConfig, TimePoint placedTime, + MonetaryAmount remainingVolume, MonetaryAmount price, + MonetaryAmount matchedVolume, ExchangeConfig::FeeType feeType) { + _availableBaseAmount -= (remainingVolume + matchedVolume); + _availableQuoteAmount += exchangeConfig.applyFee(matchedVolume.toNeutral() * price, feeType); + + if (remainingVolume == 0) { + _closedOrders.emplace_back(nextOrderId(), matchedVolume, price, placedTime, placedTime, TradeSide::kSell); + } else { + _openedOrders.emplace_back(nextOrderId(), matchedVolume, remainingVolume, price, placedTime, TradeSide::kSell); + } +} + +void MarketTraderEngineState::adjustOpenedOrderRemainingVolume(const OpenedOrder &matchedOrder, + MonetaryAmount newMatchedVolume) { + auto openedOrderIt = std::ranges::find_if( + _openedOrders, [&matchedOrder](const auto &openedOrder) { return matchedOrder.id() == openedOrder.id(); }); + + *openedOrderIt = OpenedOrder(matchedOrder.id(), matchedOrder.matchedVolume() + newMatchedVolume, + matchedOrder.remainingVolume() - newMatchedVolume, matchedOrder.price(), + matchedOrder.placedTime(), matchedOrder.side()); +} + +void MarketTraderEngineState::countMatchedPart(const ExchangeConfig &exchangeConfig, const OpenedOrder &matchedOrder, + MonetaryAmount price, MonetaryAmount newMatchedVolume, + TimePoint matchedTime) { + switch (matchedOrder.side()) { + case TradeSide::kBuy: + _availableBaseAmount += exchangeConfig.applyFee(newMatchedVolume, ExchangeConfig::FeeType::kMaker); + break; + case TradeSide::kSell: + _availableQuoteAmount += + exchangeConfig.applyFee(newMatchedVolume.toNeutral() * price, ExchangeConfig::FeeType::kMaker); + break; + default: + throw exception("Unknown trade side {}", static_cast(matchedOrder.side())); + } + + ClosedOrder newClosedOrder(matchedOrder.id(), newMatchedVolume, price, matchedOrder.placedTime(), matchedTime, + matchedOrder.side()); + + auto closedOrderIt = + std::ranges::find_if(_closedOrders.rbegin(), _closedOrders.rend(), + [&matchedOrder](const auto &closedOrder) { return closedOrder.id() == matchedOrder.id(); }); + if (closedOrderIt != _closedOrders.rend()) { + *closedOrderIt = closedOrderIt->mergeWith(newClosedOrder); + } else { + _closedOrders.push_back(std::move(newClosedOrder)); + } +} + +void MarketTraderEngineState::cancelOpenedOrder(int32_t orderId) { + const auto orderIdIt = findOpenedOrder(orderId); + adjustAvailableAmountsCancel(*orderIdIt); + _openedOrders.erase(orderIdIt); +} + +OpenedOrderVector::const_iterator MarketTraderEngineState::findOpenedOrder(int32_t orderId) { + const auto orderIdIt = std::ranges::find_if(_openedOrders, [orderId](const OpenedOrder &openedOrder) { + return FromString(openedOrder.id()) == orderId; + }); + if (orderIdIt == _openedOrders.end()) { + throw exception("Unable to find opened order id {}", orderId); + } + return orderIdIt; +} + +void MarketTraderEngineState::cancelAllOpenedOrders() { + std::ranges::for_each(_openedOrders, + [this](const OpenedOrder &openedOrder) { this->adjustAvailableAmountsCancel(openedOrder); }); + _openedOrders.clear(); +} + +void MarketTraderEngineState::adjustAvailableAmountsCancel(const OpenedOrder &openedOrder) { + switch (openedOrder.side()) { + case TradeSide::kBuy: + _availableQuoteAmount += openedOrder.remainingVolume().toNeutral() * openedOrder.price(); + break; + case TradeSide::kSell: + _availableBaseAmount += openedOrder.remainingVolume(); + break; + default: + throw exception("Unknown trade side {}", static_cast(openedOrder.side())); + } +} + +void MarketTraderEngineState::eraseClosedOpenedOrders(std::span closedOpenedOrders) { + const auto [first, last] = std::ranges::remove_if(_openedOrders, [closedOpenedOrders](const auto &openedOrder) { + return std::ranges::any_of(closedOpenedOrders, [&openedOrder](const auto &closedOpenedOrder) { + return openedOrder.id() == closedOpenedOrder.id(); + }); + }); + _openedOrders.erase(first, last); +} + +} // namespace cct \ No newline at end of file diff --git a/src/trading/common/src/market-trader-engine.cpp b/src/trading/common/src/market-trader-engine.cpp new file mode 100644 index 00000000..cee6aa97 --- /dev/null +++ b/src/trading/common/src/market-trader-engine.cpp @@ -0,0 +1,329 @@ +#include "market-trader-engine.hpp" + +#include +#include +#include + +#include "abstract-market-trader.hpp" +#include "cct_exception.hpp" +#include "cct_log.hpp" +#include "market-data-view.hpp" +#include "marketorderbook.hpp" +#include "monetaryamount.hpp" +#include "priceoptionsdef.hpp" +#include "public-trade-vector.hpp" +#include "publictrade.hpp" +#include "timestring.hpp" +#include "trade-range-stats.hpp" +#include "trader-command.hpp" + +namespace cct { + +MarketTraderEngine::MarketTraderEngine(const ExchangeConfig &exchangeConfig, Market market, + MonetaryAmount startAmountBase, MonetaryAmount startAmountQuote) + : _startAmountBase(startAmountBase), + _startAmountQuote(startAmountQuote), + _exchangeConfig(exchangeConfig), + _market(market), + _marketTraderEngineState(startAmountBase, startAmountQuote) { + if (market != this->market()) { + throw exception("Inconsistent market {} and start amounts {} & {} for MarketTraderEngine", market, startAmountBase, + startAmountQuote); + } +} + +void MarketTraderEngine::registerMarketTrader(std::unique_ptr marketTrader) { + if (_marketTrader) { + throw exception("Cannot register twice a market trader to this MarketTraderEngine"); + } + _marketTrader.swap(marketTrader); +} + +namespace { + +template +TradeRangeResultsStats ValidateRange(VectorType &vec, TimePoint earliestPossibleTime) { + using std::erase_if; + + using ObjType = std::remove_cvref_t().begin())>; + + static_assert(std::is_same_v || std::is_same_v); + + static constexpr std::string_view kObjName = std::is_same_v ? "order book" : "trade"; + + TradeRangeResultsStats stats; + + stats.nbSuccessful = static_cast(vec.size()); + + const auto nbInvalidObjects = erase_if(vec, [](const auto &obj) { return !obj.isValid(); }); + if (nbInvalidObjects != 0) { + log::error("{} {}(s) with invalid data detected", nbInvalidObjects, kObjName); + } + + const auto nbUnsortedObjectsRemoved = erase_if(vec, [&earliestPossibleTime](const auto &obj) { + if (obj.time() < earliestPossibleTime) { + return true; + } + earliestPossibleTime = obj.time(); + return false; + }); + if (nbUnsortedObjectsRemoved != 0) { + log::error("{} {}(s) are not in chronological order", nbUnsortedObjectsRemoved, kObjName); + } + + stats.nbError = nbInvalidObjects + nbUnsortedObjectsRemoved; + stats.nbSuccessful -= stats.nbError; + + return stats; +} + +} // namespace + +TradeRangeStats MarketTraderEngine::validateRange(MarketOrderBookVector &marketOrderBooks, + PublicTradeVector &publicTrades) { + TimePoint earliestPossibleTime; + if (_lastMarketOrderBook.market().isDefined()) { + earliestPossibleTime = _lastMarketOrderBook.time(); + } + + TradeRangeStats tradeRangeStats; + tradeRangeStats.marketOrderBookStats = ValidateRange(marketOrderBooks, earliestPossibleTime); + tradeRangeStats.publicTradeStats = ValidateRange(publicTrades, earliestPossibleTime); + + return tradeRangeStats; +} + +TradeRangeStats MarketTraderEngine::validateRange(MarketOrderBookVector &&marketOrderBooks, + PublicTradeVector &&publicTrades) { + const TradeRangeStats tradeRangeStats = validateRange(marketOrderBooks, publicTrades); + + if (!marketOrderBooks.empty()) { + _lastMarketOrderBook = std::move(marketOrderBooks.back()); + } + + return tradeRangeStats; +} + +TradeRangeStats MarketTraderEngine::tradeRange(MarketOrderBookVector &&marketOrderBooks, + PublicTradeVector &&publicTrades) { + if (!_marketTrader) { + throw exception("registerMarketTrader should have been called before launching the trade engine"); + } + + TradeRangeStats tradeRangeStats{{TradeRangeResultsStats{static_cast(marketOrderBooks.size()), 0}}, + TradeRangeResultsStats{static_cast(publicTrades.size()), 0}}; + + if (marketOrderBooks.empty()) { + return tradeRangeStats; + } + + log::info("[{}] at {} on {} replaying {} order books and {} trades", _marketTrader->name(), + ToString(marketOrderBooks.front().time()), _market, marketOrderBooks.size(), publicTrades.size()); + + // Rolling window of data provided to underlying market trader with data up to latest market order book. + MarketDataView marketDataView(marketOrderBooks.data(), publicTrades.data(), + publicTrades.data() + publicTrades.size()); + + for (const MarketOrderBook &marketOrderBook : marketOrderBooks) { + // First check opened orders status with new market order book data that may match some + checkOpenedOrdersMatching(marketOrderBook); + + // We expect market data (order books and trades) to be sorted by time. + // Advance the market data view iterator until including all data until last market order book time stamp. + marketDataView.advanceUntil(marketOrderBook.time()); + + // Call the user algorithm trading engine and retrieve its decision for next move + const TraderCommand traderCommand = _marketTrader->trade(marketDataView); + + switch (traderCommand.type()) { + case TraderCommand::Type::kWait: + break; + case TraderCommand::Type::kBuy: { + const MonetaryAmount from = _marketTraderEngineState.computeBuyFrom(traderCommand); + + if (from != 0) { + // Attempt to place an order without any available amount, do nothing instead + buy(marketOrderBook, from, traderCommand.priceStrategy()); + } + break; + } + case TraderCommand::Type::kSell: { + const MonetaryAmount volume = _marketTraderEngineState.computeSellVolume(traderCommand); + + if (volume != 0) { + // Attempt to place an order without any available amount, do nothing instead + sell(marketOrderBook, volume, traderCommand.priceStrategy()); + } + break; + } + case TraderCommand::Type::kUpdatePrice: + updatePrice(marketOrderBook, traderCommand); + break; + case TraderCommand::Type::kCancel: + cancelCommand(traderCommand.orderId()); + break; + default: + throw exception("Unsupported trader command {}", static_cast(traderCommand.type())); + } + } + + _lastMarketOrderBook = std::move(marketOrderBooks.back()); + + return tradeRangeStats; +} + +MarketTradingResult MarketTraderEngine::finalizeAndComputeResult() { + if (!_marketTrader) { + throw exception("registerMarketTrader should have been called before computing results"); + } + + _marketTraderEngineState.cancelAllOpenedOrders(); + + // How to compute gain / losses ? + // Let's say we have {x1 XXX + y1 YYY} at the beginning, XXX-YYY being the market, + // and {x2 XXX + y2 YYY} at the end. + // The idea is that we speculate on the YYY currency on this market (we want to increase our YYY amount). + // The formula used to compute gains / losses is the following: + // (y2 - y1) YYY + conversion((x2 - x1) XXX)->YYY + at market price of the last market order book. + + MonetaryAmount quoteAmountDelta = _marketTraderEngineState.availableQuoteAmount() - _startAmountQuote; + MonetaryAmount baseAmountDelta = _marketTraderEngineState.availableBaseAmount() - _startAmountBase; + + if (_lastMarketOrderBook.market().isNeutral()) { + log::debug("Calling finalize on a market trader engine that has not been run"); + } else { + auto [_, avgPrice] = _lastMarketOrderBook.avgPriceAndMatchedAmountTaker(baseAmountDelta.abs()); + + quoteAmountDelta += baseAmountDelta.toNeutral() * avgPrice; + } + + const auto closedOrdersSpan = _marketTraderEngineState.closedOrders(); + + return MarketTradingResult(_marketTrader->name(), _startAmountBase, _startAmountQuote, quoteAmountDelta, + ClosedOrderVector(closedOrdersSpan.begin(), closedOrdersSpan.end())); +} + +void MarketTraderEngine::buy(const MarketOrderBook &marketOrderBook, MonetaryAmount from, PriceStrategy priceStrategy) { + const auto ts = marketOrderBook.time(); + + switch (priceStrategy) { + case PriceStrategy::kMaker: { + const MonetaryAmount price = marketOrderBook.highestBidPrice(); + const MonetaryAmount remainingVolume(from / price, _market.base()); + constexpr MonetaryAmount matchedVolume; + + _marketTraderEngineState.placeBuyOrder(_exchangeConfig, ts, remainingVolume, price, matchedVolume, from, + ExchangeConfig::FeeType::kMaker); + break; + } + case PriceStrategy::kNibble: { + const MonetaryAmount price = marketOrderBook.lowestAskPrice(); + const MonetaryAmount volume(from / price, _market.base()); + const MonetaryAmount matchedVolume = std::min(marketOrderBook.amountAtAskPrice(), volume); + const MonetaryAmount remainingVolume = volume - matchedVolume; + + _marketTraderEngineState.placeBuyOrder(_exchangeConfig, ts, remainingVolume, price, matchedVolume, from, + ExchangeConfig::FeeType::kTaker); + break; + } + case PriceStrategy::kTaker: { + const auto [totalMatchedAmount, avgPrice] = marketOrderBook.avgPriceAndMatchedAmountTaker(from); + if (totalMatchedAmount != 0) { + constexpr MonetaryAmount remainingVolume; + + _marketTraderEngineState.placeBuyOrder(_exchangeConfig, ts, remainingVolume, avgPrice, totalMatchedAmount, from, + ExchangeConfig::FeeType::kTaker); + } + break; + } + default: + throw exception("Unsupported price strategy {}", static_cast(priceStrategy)); + } +} + +void MarketTraderEngine::sell(const MarketOrderBook &marketOrderBook, MonetaryAmount volume, + PriceStrategy priceStrategy) { + switch (priceStrategy) { + case PriceStrategy::kMaker: { + const MonetaryAmount price = marketOrderBook.lowestAskPrice(); + constexpr MonetaryAmount matchedVolume; + + _marketTraderEngineState.placeSellOrder(_exchangeConfig, marketOrderBook.time(), volume, price, matchedVolume, + ExchangeConfig::FeeType::kMaker); + break; + } + case PriceStrategy::kNibble: { + const MonetaryAmount price = marketOrderBook.highestBidPrice(); + const MonetaryAmount matchedVolume = std::min(marketOrderBook.amountAtBidPrice(), volume); + + _marketTraderEngineState.placeSellOrder(_exchangeConfig, marketOrderBook.time(), volume - matchedVolume, price, + matchedVolume, ExchangeConfig::FeeType::kTaker); + break; + } + case PriceStrategy::kTaker: { + const auto [totalMatchedAmount, avgPrice] = marketOrderBook.avgPriceAndMatchedAmountTaker(volume); + + if (totalMatchedAmount != 0) { + constexpr MonetaryAmount remainingVolume; + + _marketTraderEngineState.placeSellOrder(_exchangeConfig, marketOrderBook.time(), remainingVolume, avgPrice, + totalMatchedAmount, ExchangeConfig::FeeType::kTaker); + } + break; + } + default: + throw exception("Unsupported price strategy {}", static_cast(priceStrategy)); + } +} + +void MarketTraderEngine::updatePrice(const MarketOrderBook &marketOrderBook, TraderCommand traderCommand) { + const auto orderIdIt = _marketTraderEngineState.findOpenedOrder(traderCommand.orderId()); + MonetaryAmount remainingAmount = orderIdIt->remainingVolume(); + TradeSide tradeSide = orderIdIt->side(); + MonetaryAmount price = orderIdIt->price(); + + _marketTraderEngineState.cancelOpenedOrder(traderCommand.orderId()); + + switch (tradeSide) { + case TradeSide::kBuy: + buy(marketOrderBook, remainingAmount.toNeutral() * price, traderCommand.priceStrategy()); + break; + case TradeSide::kSell: + sell(marketOrderBook, remainingAmount, traderCommand.priceStrategy()); + break; + default: + throw exception("Unsupported trade side"); + } +} + +void MarketTraderEngine::cancelCommand(int32_t orderId) { + if (orderId == TraderCommand::kAllOrdersId) { + _marketTraderEngineState.cancelAllOpenedOrders(); + } else { + _marketTraderEngineState.cancelOpenedOrder(orderId); + } +} + +void MarketTraderEngine::checkOpenedOrdersMatching(const MarketOrderBook &marketOrderBook) { + _newlyClosedOrders.clear(); + for (const OpenedOrder &openedOrder : _marketTraderEngineState.openedOrders()) { + const auto [newMatchedVolume, avgPrice] = marketOrderBook.avgPriceAndMatchedVolume( + openedOrder.side(), openedOrder.remainingVolume(), openedOrder.price()); + if (newMatchedVolume == 0) { + continue; + } + + _marketTraderEngineState.countMatchedPart(_exchangeConfig, openedOrder, avgPrice, newMatchedVolume, + marketOrderBook.time()); + + if (newMatchedVolume == openedOrder.remainingVolume()) { + _newlyClosedOrders.push_back(openedOrder); + } else { + _marketTraderEngineState.adjustOpenedOrderRemainingVolume(openedOrder, newMatchedVolume); + } + } + + _marketTraderEngineState.eraseClosedOpenedOrders(_newlyClosedOrders); +} + +} // namespace cct diff --git a/src/trading/common/src/market-trading-result.cpp b/src/trading/common/src/market-trading-result.cpp new file mode 100644 index 00000000..ad7e2f59 --- /dev/null +++ b/src/trading/common/src/market-trading-result.cpp @@ -0,0 +1,20 @@ +#include "market-trading-result.hpp" + +#include +#include + +#include "exchangeprivateapitypes.hpp" +#include "monetaryamount.hpp" + +namespace cct { + +MarketTradingResult::MarketTradingResult(std::string_view algorithmName, MonetaryAmount startBaseAmount, + MonetaryAmount startQuoteAmount, MonetaryAmount quoteAmountDelta, + ClosedOrderVector matchedOrders) + : _algorithmName(algorithmName), + _startBaseAmount(startBaseAmount), + _startQuoteAmount(startQuoteAmount), + _quoteAmountDelta(quoteAmountDelta), + _matchedOrders(std::move(matchedOrders)) {} + +} // namespace cct \ No newline at end of file diff --git a/src/trading/common/src/trader-command.cpp b/src/trading/common/src/trader-command.cpp new file mode 100644 index 00000000..e7ddbf63 --- /dev/null +++ b/src/trading/common/src/trader-command.cpp @@ -0,0 +1,62 @@ +#include "trader-command.hpp" + +#include + +#include "cct_exception.hpp" +#include "priceoptionsdef.hpp" +#include "stringhelpers.hpp" +#include "tradeside.hpp" + +namespace cct { +TraderCommand::TraderCommand(Type type, int32_t orderId, int8_t amountIntensityPercentage, PriceStrategy priceStrategy) + : _orderId(orderId), + _type(type), + _amountIntensityPercentage(amountIntensityPercentage), + _priceStrategy(priceStrategy) {} + +TraderCommand TraderCommand::Wait() { return TraderCommand(Type::kWait, kAllOrdersId, 0, PriceStrategy::kMaker); } + +TraderCommand TraderCommand::Place(TradeSide tradeSide, int8_t amountIntensityPercentage, PriceStrategy priceStrategy) { + if (amountIntensityPercentage > 100 || amountIntensityPercentage <= 0) { + throw exception("Invalid amountIntensityPercentage {}", amountIntensityPercentage); + } + Type type; + switch (tradeSide) { + case TradeSide::kBuy: + type = Type::kBuy; + break; + case TradeSide::kSell: + type = Type::kSell; + break; + default: + throw exception("Unexpected trade side"); + } + return TraderCommand(type, kAllOrdersId, amountIntensityPercentage, priceStrategy); +} + +TraderCommand TraderCommand::Cancel(OrderIdView orderId) { + int32_t orderIdInt; + if (!orderId.empty()) { + orderIdInt = FromString(orderId); + } else { + orderIdInt = kAllOrdersId; + } + return TraderCommand(Type::kCancel, orderIdInt, 0, PriceStrategy::kMaker); +} + +TraderCommand TraderCommand::UpdatePrice(OrderIdView orderId, PriceStrategy priceStrategy) { + return TraderCommand(Type::kUpdatePrice, FromString(orderId), 100, priceStrategy); +} + +TradeSide TraderCommand::tradeSide() const { + switch (_type) { + case Type::kBuy: + return TradeSide::kBuy; + case Type::kSell: + return TradeSide::kSell; + default: + throw exception("Unexpected trade command type for trade side"); + } +} + +} // namespace cct \ No newline at end of file diff --git a/vcpkg.json b/vcpkg.json index 809e67b1..35ffefcf 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -1,6 +1,7 @@ { "dependencies": [ "openssl", + "zlib", "curl" ] -} +} \ No newline at end of file