From ddf574c88793292148a3ee25c2f9e4f3ebae3d08 Mon Sep 17 00:00:00 2001 From: Crypto Chassis Date: Mon, 29 May 2023 10:20:10 -0700 Subject: [PATCH 1/2] dev: get ready for C++17 --- README.md | 2 +- app/CMakeLists.txt | 2 +- app/script/download_historical_market_data.py | 132 ------------------ app/script/requirements.txt | 1 - .../single_order_execution/config.env.example | 5 +- app/src/spot_market_making/config.env.example | 5 +- binding/CMakeLists.txt | 2 +- example/CMakeLists.txt | 2 +- performance/CMakeLists.txt | 2 +- test/CMakeLists.txt | 2 +- 10 files changed, 8 insertions(+), 147 deletions(-) delete mode 100755 app/script/download_historical_market_data.py delete mode 100644 app/script/requirements.txt diff --git a/README.md b/README.md index 549319d1..66884879 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ Notification to Maintainers and Developers: we are aiming at an effort to transi ### C++ * This library is header-only. * Example CMake: example/CMakeLists.txt. -* Require C++14 and OpenSSL. +* Require C++14 (prefer C++17) and OpenSSL. * Macros in the compiler command line: * Define service enablement macro such as `CCAPI_ENABLE_SERVICE_MARKET_DATA`, `CCAPI_ENABLE_SERVICE_EXECUTION_MANAGEMENT`, `CCAPI_ENABLE_SERVICE_FIX`, etc. and exchange enablement macros such as `CCAPI_ENABLE_EXCHANGE_COINBASE`, etc. These macros can be found at the top of [`include/ccapi_cpp/ccapi_session.h`](include/ccapi_cpp/ccapi_session.h). * Include directories: diff --git a/app/CMakeLists.txt b/app/CMakeLists.txt index 70de13dd..ca247256 100644 --- a/app/CMakeLists.txt +++ b/app/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.14) set(NAME app) project(${NAME}) if(NOT "${CMAKE_CXX_STANDARD}") - set(CMAKE_CXX_STANDARD 14) + set(CMAKE_CXX_STANDARD 17) endif() message(STATUS "CMAKE_CXX_STANDARD: ${CMAKE_CXX_STANDARD}") if(NOT APPLE AND NOT MSVC) diff --git a/app/script/download_historical_market_data.py b/app/script/download_historical_market_data.py deleted file mode 100755 index c668486d..00000000 --- a/app/script/download_historical_market_data.py +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import gzip -import os -import pathlib -import time -from datetime import datetime, timedelta - -import requests -from requests.adapters import HTTPAdapter -from urllib3.util.retry import Retry - - -class TimeoutHTTPAdapter(HTTPAdapter): - def __init__(self, *args, **kwargs): - self.timeout = 5 - if "timeout" in kwargs: - self.timeout = kwargs["timeout"] - del kwargs["timeout"] - super().__init__(*args, **kwargs) - - def send(self, request, **kwargs): - timeout = kwargs.get("timeout") - if timeout is None: - kwargs["timeout"] = self.timeout - return super().send(request, **kwargs) - - -if __name__ == "__main__": - argumentParser = argparse.ArgumentParser() - - argumentParser.add_argument("--exchange", required=True, type=str, help="The exchange, e.g. coinbase.") - argumentParser.add_argument("--base-asset", required=True, type=str, help="The base asset, e.g. btc.") - argumentParser.add_argument("--quote-asset", required=True, type=str, help="The quote asset, e.g. usd.") - argumentParser.add_argument("--start-date", required=True, type=str, help="The start date, e.g. 2021-08-22.") - argumentParser.add_argument( - "--end-date", - required=True, - type=str, - help="The end date, e.g. 2021-08-23. Exclusive.", - ) - argumentParser.add_argument( - "--historical-market-data-directory", - required=True, - type=str, - help="The directory in which historical market data files are saved.", - ) - argumentParser.add_argument( - "--historical-market-data-file-prefix", - required=False, - type=str, - default="", - help="This value specifies the name prefix of the files in which historical market data are saved.", - ) - argumentParser.add_argument( - "--historical-market-data-file-suffix", - required=False, - type=str, - default="", - help="This value specifies the name suffix of the files in which historical market data are saved.", - ) - argumentParser.add_argument( - "--depth", - required=False, - type=int, - default=1, - choices=[1, 10], - help="The depth of market depth data.", - ) - - args = argumentParser.parse_args() - - exchange = args.exchange - baseAsset = args.base_asset.lower() - quoteAsset = args.quote_asset.lower() - startDate = datetime.strptime(args.start_date, "%Y-%m-%d").date() - endDate = datetime.strptime(args.end_date, "%Y-%m-%d").date() - currentDate = startDate - historicalMarketDataDirectory = args.historical_market_data_directory - historicalMarketDataFilePrefix = args.historical_market_data_file_prefix - historicalMarketDataFileSuffix = args.historical_market_data_file_suffix - depth = args.depth - pathlib.Path(historicalMarketDataDirectory).mkdir(parents=True, exist_ok=True) - urlBase = "https://api.cryptochassis.com/v1" - session = requests.Session() - retries = Retry(total=10, backoff_factor=1, status_forcelist=[429, 500, 502, 503, 504]) - session.mount("https://", TimeoutHTTPAdapter(max_retries=retries)) - tradeCsvHeader = "time_seconds,price,size,is_buyer_maker\n" - while currentDate < endDate: - for dataType in ["market-depth", "trade"]: - fileName = f"{historicalMarketDataFilePrefix}{exchange}__{baseAsset}-{quoteAsset}__{currentDate.isoformat()}__{dataType}{historicalMarketDataFileSuffix}" - fileNameWithDir = f"{historicalMarketDataDirectory}/{fileName}" - tmpFileNameWithDir = f"{historicalMarketDataDirectory}/tmp__{fileName}" - if not pathlib.Path(f"{fileNameWithDir}.csv").is_file(): - print(f"Start download data for {dataType}, {exchange}, {baseAsset}-{quoteAsset}, {currentDate.isoformat()}.") - requestUrl = f"{urlBase}/{dataType}/{exchange}/{baseAsset}-{quoteAsset}?startTime={currentDate.isoformat()}" - if dataType == "market-depth": - requestUrl += f"&depth={depth}" - urls = session.get(requestUrl).json()["urls"] - time.sleep(0.1) - if not urls: - print(f"Data cannot be found on server. Skip download.") - continue - fileUrl = urls[0]["url"] - with session.get(fileUrl, stream=True) as r: - with open(f"{fileNameWithDir}.csv.gz", "wb") as f: - for chunk in r.iter_content(chunk_size=1024): - f.write(chunk) - with open(f"{tmpFileNameWithDir}.csv", "wb+") as fOut: - with gzip.open(f"{fileNameWithDir}.csv.gz", "rb") as fIn: - fOut.writelines(fIn) - os.remove(f"{fileNameWithDir}.csv.gz") - - if dataType == "market-depth": - os.rename(f"{tmpFileNameWithDir}.csv", f"{fileNameWithDir}.csv") - else: - with open(f"{tmpFileNameWithDir}.csv") as fIn: - firstLine = fIn.readline() - if firstLine == tradeCsvHeader: - os.rename(f"{tmpFileNameWithDir}.csv", f"{fileNameWithDir}.csv") - else: - with open(f"{fileNameWithDir}.csv", "w") as fOut: - fOut.write(tradeCsvHeader) - with open(f"{tmpFileNameWithDir}.csv") as fIn: - next(fIn) - for line in fIn: - splitted = tuple(line.split(",")) - fOut.write( - ",".join((splitted[0] + "." + splitted[1].zfill(9).rstrip("0") if splitted[1] != "0" else splitted[0],) + splitted[2:5]) - ) - os.remove(f"{tmpFileNameWithDir}.csv") - currentDate += timedelta(days=1) diff --git a/app/script/requirements.txt b/app/script/requirements.txt deleted file mode 100644 index f2293605..00000000 --- a/app/script/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -requests diff --git a/app/src/single_order_execution/config.env.example b/app/src/single_order_execution/config.env.example index 21af54e7..27a1c201 100644 --- a/app/src/single_order_execution/config.env.example +++ b/app/src/single_order_execution/config.env.example @@ -24,10 +24,7 @@ # time_seconds,price,size,is_buyer_maker # 1625097621.647,2278.15,0.0409,1 # ... -# You can choose to get the data from any vendor. If you choose https://github.com/crypto-chassis/cryptochassis-data-api-docs, -# you can use a convenience script app/script/download_historical_market_data.py. For example, -# python3 download_historical_market_data.py --exchange gemini --base-asset eth --quote-asset usd --start-date 2021-07-01 -# --end-date 2021-07-03 --historical-market-data-directory --market-depth 10 +# You can choose to get the data from any vendor. # 'live': Live trade. diff --git a/app/src/spot_market_making/config.env.example b/app/src/spot_market_making/config.env.example index 4fff9ec4..04e5ebfc 100644 --- a/app/src/spot_market_making/config.env.example +++ b/app/src/spot_market_making/config.env.example @@ -28,10 +28,7 @@ # time_seconds,price,size,is_buyer_maker # 1625097621.647,2278.15,0.0409,1 # ... -# You can choose to get the data from any vendor. If you choose https://github.com/crypto-chassis/cryptochassis-data-api-docs, -# you can use a convenience script app/script/download_historical_market_data.py. For example, -# python3 download_historical_market_data.py --exchange gemini --base-asset eth --quote-asset usd --start-date 2021-07-01 -# --end-date 2021-07-03 --historical-market-data-directory +# You can choose to get the data from any vendor. # 'live': Live trade. diff --git a/binding/CMakeLists.txt b/binding/CMakeLists.txt index c44c5cce..d05eefd3 100644 --- a/binding/CMakeLists.txt +++ b/binding/CMakeLists.txt @@ -76,7 +76,7 @@ option(BUILD_PYTHON "Build Python Library" OFF) option(INSTALL_PYTHON "Install Python Library" OFF) message(STATUS "Build Python: ${BUILD_PYTHON}") -set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD 17) if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread") endif() diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index 370d4fd4..5f64d73f 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.14) set(NAME example) project(${NAME}) -set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD 17) if(NOT APPLE AND NOT MSVC) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread") endif() diff --git a/performance/CMakeLists.txt b/performance/CMakeLists.txt index 7fc9fc83..592c5639 100644 --- a/performance/CMakeLists.txt +++ b/performance/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.14) set(NAME performance) project(${NAME}) -set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD 17) if(NOT APPLE AND NOT MSVC) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread") endif() diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index fd5fb9a6..2b355fd2 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.14) set(NAME test) project(${NAME}) -set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD 17) if(NOT APPLE AND NOT MSVC) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread") endif() From 6afa7d707a10d59301946b7bb6357032fba1094e Mon Sep 17 00:00:00 2001 From: Crypto Chassis Date: Mon, 12 Jun 2023 19:38:59 -0700 Subject: [PATCH 2/2] dev: fix websocket connection issues when using boost beast websocket --- app/CMakeLists.txt | 1 + binding/CMakeLists.txt | 1 + example/CMakeLists.txt | 1 + include/ccapi_cpp/ccapi_ws_connection.h | 70 ++++++++++++------- .../ccapi_execution_management_service.h | 2 +- ...pi_execution_management_service_ascendex.h | 2 +- ...xecution_management_service_binance_base.h | 2 +- ...pi_execution_management_service_bitstamp.h | 2 +- ...agement_service_gateio_perpetual_futures.h | 2 +- ...capi_execution_management_service_gemini.h | 8 +-- ...capi_execution_management_service_kraken.h | 2 +- ...execution_management_service_kucoin_base.h | 3 +- .../ccapi_execution_management_service_mexc.h | 2 +- .../service/ccapi_market_data_service.h | 3 +- .../ccapi_market_data_service_gemini.h | 4 +- .../ccapi_market_data_service_kucoin_base.h | 2 +- include/ccapi_cpp/service/ccapi_service.h | 42 ++++++----- performance/CMakeLists.txt | 1 + test/CMakeLists.txt | 1 + 19 files changed, 90 insertions(+), 61 deletions(-) diff --git a/app/CMakeLists.txt b/app/CMakeLists.txt index ca247256..0217c5e7 100644 --- a/app/CMakeLists.txt +++ b/app/CMakeLists.txt @@ -37,5 +37,6 @@ if (WIN32) set(ADDITIONAL_LINK_LIBRARIES ws2_32) endif() link_libraries(OpenSSL::Crypto OpenSSL::SSL ${ADDITIONAL_LINK_LIBRARIES}) +add_compile_options(-Wno-deprecated) add_subdirectory(src/spot_market_making) add_subdirectory(src/single_order_execution) diff --git a/binding/CMakeLists.txt b/binding/CMakeLists.txt index d05eefd3..f3744501 100644 --- a/binding/CMakeLists.txt +++ b/binding/CMakeLists.txt @@ -105,4 +105,5 @@ include(UseSWIG) if(BUILD_TEST) include(CTest) endif() +add_compile_options(-Wno-deprecated) add_subdirectory(python) diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index 5f64d73f..a5ac106c 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -31,6 +31,7 @@ if (WIN32) set(ADDITIONAL_LINK_LIBRARIES ws2_32) endif() link_libraries(OpenSSL::Crypto OpenSSL::SSL ${ADDITIONAL_LINK_LIBRARIES}) +add_compile_options(-Wno-deprecated) add_subdirectory(src/market_data_simple_request) add_subdirectory(src/market_data_simple_subscription) add_subdirectory(src/market_data_advanced_request) diff --git a/include/ccapi_cpp/ccapi_ws_connection.h b/include/ccapi_cpp/ccapi_ws_connection.h index dddc6c8a..a7a550e6 100644 --- a/include/ccapi_cpp/ccapi_ws_connection.h +++ b/include/ccapi_cpp/ccapi_ws_connection.h @@ -93,29 +93,7 @@ class WsConnection CCAPI_FINAL { this->correlationIdList.reserve(subscriptionList.size()); std::transform(subscriptionList.cbegin(), subscriptionList.cend(), this->correlationIdList.begin(), [](Subscription subscription) { return subscription.getCorrelationId(); }); - auto splitted1 = UtilString::split(url, "://"); - auto foundSlash = splitted1.at(1).find_first_of('/'); - auto foundQuestionMark = splitted1.at(1).find_first_of('?'); - if (foundSlash == std::string::npos && foundQuestionMark == std::string::npos) { - this->path = "/"; - } else if (foundSlash == std::string::npos && foundQuestionMark != std::string::npos) { - this->path = "/" + splitted1.at(1).substr(foundQuestionMark); - } else if (foundSlash != std::string::npos && foundQuestionMark == std::string::npos) { - this->path = splitted1.at(1).substr(foundSlash); - } else { - this->path = splitted1.at(1).substr(foundSlash); - } - auto splitted2 = UtilString::split(UtilString::split(splitted1.at(1), "/").at(0), ":"); - this->host = splitted2.at(0); - if (splitted2.size() == 2) { - this->port = splitted2.at(1); - } else { - if (splitted1.at(0) == "https" || splitted1.at(0) == "wss") { - this->port = CCAPI_HTTPS_PORT_DEFAULT; - } else { - this->port = CCAPI_HTTP_PORT_DEFAULT; - } - } + this->setUrlParts(); } WsConnection() {} std::string toString() const { @@ -127,7 +105,10 @@ class WsConnection CCAPI_FINAL { oss << streamPtr; std::string output = "WsConnection [id = " + id + ", url = " + url + ", group = " + group + ", subscriptionList = " + ccapi::toString(subscriptionList) + ", credential = " + ccapi::toString(shortCredential) + ", status = " + statusToString(status) + - ", headers = " + ccapi::toString(headers) + ", streamPtr = " + oss.str() + "]"; + ", headers = " + ccapi::toString(headers) + ", streamPtr = " + oss.str() + ", remoteCloseCode = " + std::to_string(remoteCloseCode) + + ", remoteCloseReason = " + std::string(remoteCloseReason.reason.c_str()) + + ", hostHttpHeaderValue = " + ccapi::toString(hostHttpHeaderValue) + ", path = " + ccapi::toString(path) + + ", host = " + ccapi::toString(host) + ", port = " + ccapi::toString(port) + "]"; return output; } enum class Status { @@ -164,8 +145,43 @@ class WsConnection CCAPI_FINAL { } return output; } + std::string getUrl() const { return url; } + void setUrl(const std::string& url) { + this->url = url; + this->setUrlParts(); + } + void setUrlParts() { + auto splitted1 = UtilString::split(url, "://"); + if (splitted1.size() >= 2) { + auto foundSlash = splitted1.at(1).find_first_of('/'); + auto foundQuestionMark = splitted1.at(1).find_first_of('?'); + if (foundSlash == std::string::npos && foundQuestionMark == std::string::npos) { + this->path = "/"; + } else if (foundSlash == std::string::npos && foundQuestionMark != std::string::npos) { + this->path = "/" + splitted1.at(1).substr(foundQuestionMark); + } else if (foundSlash != std::string::npos && foundQuestionMark == std::string::npos) { + this->path = splitted1.at(1).substr(foundSlash); + } else { + this->path = splitted1.at(1).substr(foundSlash); + } + auto splitted2 = UtilString::split(UtilString::split(splitted1.at(1), "/").at(0), ":"); + this->host = splitted2.at(0); + if (splitted2.size() == 2) { + this->port = splitted2.at(1); + } else { + if (splitted1.at(0) == "https" || splitted1.at(0) == "wss") { + this->port = CCAPI_HTTPS_PORT_DEFAULT; + } else { + this->port = CCAPI_HTTP_PORT_DEFAULT; + } + } + } + } + void appendUrlPart(const std::string& urlPart) { + this->url += urlPart; + this->setUrlParts(); + } std::string id; - std::string url; std::string group; std::vector subscriptionList; std::vector correlationIdList; @@ -179,6 +195,10 @@ class WsConnection CCAPI_FINAL { std::string path; std::string host; std::string port; +#ifndef CCAPI_EXPOSE_INTERNAL + private: +#endif + std::string url; }; } /* namespace ccapi */ #endif diff --git a/include/ccapi_cpp/service/ccapi_execution_management_service.h b/include/ccapi_cpp/service/ccapi_execution_management_service.h index 0b1a9a90..91f94654 100644 --- a/include/ccapi_cpp/service/ccapi_execution_management_service.h +++ b/include/ccapi_cpp/service/ccapi_execution_management_service.h @@ -61,7 +61,7 @@ class ExecutionManagementService : public Service { #else std::shared_ptr>> streamPtr(nullptr); try { - streamPtr = that->createStream>>(that->serviceContextPtr->ioContextPtr, that->serviceContextPtr->sslContextPtr, that->hostWs); + streamPtr = that->createWsStream(that->serviceContextPtr->ioContextPtr, that->serviceContextPtr->sslContextPtr); } catch (const beast::error_code& ec) { CCAPI_LOGGER_TRACE("fail"); that->onError(Event::Type::SUBSCRIPTION_STATUS, Message::Type::SUBSCRIPTION_FAILURE, ec, "create stream", {subscription.getCorrelationId()}); diff --git a/include/ccapi_cpp/service/ccapi_execution_management_service_ascendex.h b/include/ccapi_cpp/service/ccapi_execution_management_service_ascendex.h index f7272847..44aad7df 100644 --- a/include/ccapi_cpp/service/ccapi_execution_management_service_ascendex.h +++ b/include/ccapi_cpp/service/ccapi_execution_management_service_ascendex.h @@ -369,7 +369,7 @@ class ExecutionManagementServiceAscendex : public ExecutionManagementService { #else std::shared_ptr>> streamPtr(nullptr); try { - streamPtr = that->createStream>>(that->serviceContextPtr->ioContextPtr, that->serviceContextPtr->sslContextPtr, that->hostWs); + streamPtr = that->createWsStream(that->serviceContextPtr->ioContextPtr, that->serviceContextPtr->sslContextPtr); } catch (const beast::error_code& ec) { CCAPI_LOGGER_TRACE("fail"); that->onError(Event::Type::SUBSCRIPTION_STATUS, Message::Type::SUBSCRIPTION_FAILURE, ec, "create stream", {subscription.getCorrelationId()}); diff --git a/include/ccapi_cpp/service/ccapi_execution_management_service_binance_base.h b/include/ccapi_cpp/service/ccapi_execution_management_service_binance_base.h index 18849d20..36479bb2 100644 --- a/include/ccapi_cpp/service/ccapi_execution_management_service_binance_base.h +++ b/include/ccapi_cpp/service/ccapi_execution_management_service_binance_base.h @@ -189,7 +189,7 @@ class ExecutionManagementServiceBinanceBase : public ExecutionManagementService document.Parse(body.c_str()); std::string listenKey = document["listenKey"].GetString(); std::string url = that->baseUrlWs + "/" + listenKey; - wsConnectionPtr->url = url; + wsConnectionPtr->setUrl(url); that->connect(wsConnectionPtr); that->extraPropertyByConnectionIdMap[wsConnectionPtr->id].insert({ {"listenKey", listenKey}, diff --git a/include/ccapi_cpp/service/ccapi_execution_management_service_bitstamp.h b/include/ccapi_cpp/service/ccapi_execution_management_service_bitstamp.h index 7f162856..c217f498 100644 --- a/include/ccapi_cpp/service/ccapi_execution_management_service_bitstamp.h +++ b/include/ccapi_cpp/service/ccapi_execution_management_service_bitstamp.h @@ -367,7 +367,7 @@ class ExecutionManagementServiceBitstamp : public ExecutionManagementService { if (document.HasMember("token") && document.HasMember("user_id")) { std::string token = document["token"].GetString(); std::string userId = document["user_id"].GetString(); - wsConnectionPtr->url = that->baseUrlWs; + wsConnectionPtr->setUrl(that->baseUrlWs); that->connect(wsConnectionPtr); that->extraPropertyByConnectionIdMap[wsConnectionPtr->id].insert({ {"token", token}, diff --git a/include/ccapi_cpp/service/ccapi_execution_management_service_gateio_perpetual_futures.h b/include/ccapi_cpp/service/ccapi_execution_management_service_gateio_perpetual_futures.h index 319d68ce..fd6478ba 100644 --- a/include/ccapi_cpp/service/ccapi_execution_management_service_gateio_perpetual_futures.h +++ b/include/ccapi_cpp/service/ccapi_execution_management_service_gateio_perpetual_futures.h @@ -111,7 +111,7 @@ class ExecutionManagementServiceGateioPerpetualFutures : public ExecutionManagem #else std::shared_ptr>> streamPtr(nullptr); try { - streamPtr = that->createStream>>(that->serviceContextPtr->ioContextPtr, that->serviceContextPtr->sslContextPtr, that->hostWs); + streamPtr = that->createWsStream(that->serviceContextPtr->ioContextPtr, that->serviceContextPtr->sslContextPtr); } catch (const beast::error_code& ec) { CCAPI_LOGGER_TRACE("fail"); that->onError(Event::Type::SUBSCRIPTION_STATUS, Message::Type::SUBSCRIPTION_FAILURE, ec, "create stream", {subscription.getCorrelationId()}); diff --git a/include/ccapi_cpp/service/ccapi_execution_management_service_gemini.h b/include/ccapi_cpp/service/ccapi_execution_management_service_gemini.h index 8ccc343b..5db718ec 100644 --- a/include/ccapi_cpp/service/ccapi_execution_management_service_gemini.h +++ b/include/ccapi_cpp/service/ccapi_execution_management_service_gemini.h @@ -315,16 +315,16 @@ class ExecutionManagementServiceGemini : public ExecutionManagementService { credential = this->credentialDefault; } auto apiKey = mapGetWithDefault(credential, this->apiKeyName); - wsConnectionPtr->url += "?heartbeat=true"; + wsConnectionPtr->appendUrlPart("?heartbeat=true"); if (fieldSet == std::set({CCAPI_EM_PRIVATE_TRADE})) { - wsConnectionPtr->url += "&eventTypeFilter=fill"; + wsConnectionPtr->appendUrlPart("&eventTypeFilter=fill"); } if (!instrumentSet.empty()) { for (const auto& instrument : instrumentSet) { - wsConnectionPtr->url += "&symbolFilter=" + instrument; + wsConnectionPtr->appendUrlPart("&symbolFilter=" + instrument); } } - wsConnectionPtr->url += "&apiSessionFilter=" + apiKey; + wsConnectionPtr->appendUrlPart("&apiSessionFilter=" + apiKey); wsConnectionPtr->headers.insert({"X-GEMINI-APIKEY", apiKey}); int64_t nonce = std::chrono::duration_cast(now.time_since_epoch()).count(); std::string payload = R"({"request":"/v1/order/events","nonce":)" + std::to_string(nonce) + "}"; diff --git a/include/ccapi_cpp/service/ccapi_execution_management_service_kraken.h b/include/ccapi_cpp/service/ccapi_execution_management_service_kraken.h index 4a1b80fd..f57c1cd0 100644 --- a/include/ccapi_cpp/service/ccapi_execution_management_service_kraken.h +++ b/include/ccapi_cpp/service/ccapi_execution_management_service_kraken.h @@ -361,7 +361,7 @@ class ExecutionManagementServiceKraken : public ExecutionManagementService { document.Parse(body.c_str()); if (document.HasMember("result") && document["result"].HasMember("token")) { std::string token = document["result"]["token"].GetString(); - wsConnectionPtr->url = that->baseUrlWs; + wsConnectionPtr->setUrl(that->baseUrlWs); that->connect(wsConnectionPtr); that->extraPropertyByConnectionIdMap[wsConnectionPtr->id].insert({ {"token", token}, diff --git a/include/ccapi_cpp/service/ccapi_execution_management_service_kucoin_base.h b/include/ccapi_cpp/service/ccapi_execution_management_service_kucoin_base.h index 094ac2ff..cd667b96 100644 --- a/include/ccapi_cpp/service/ccapi_execution_management_service_kucoin_base.h +++ b/include/ccapi_cpp/service/ccapi_execution_management_service_kucoin_base.h @@ -105,7 +105,8 @@ class ExecutionManagementServiceKucoinBase : public ExecutionManagementService { urlWebsocketBase += std::string(instanceServer["endpoint"].GetString()); urlWebsocketBase += "?token="; urlWebsocketBase += std::string(document["data"]["token"].GetString()); - wsConnectionPtr->url = urlWebsocketBase; + wsConnectionPtr->setUrl(urlWebsocketBase); + std::cout << wsConnectionPtr->toString() << std::endl; that->connect(wsConnectionPtr); that->extraPropertyByConnectionIdMap[wsConnectionPtr->id].insert({ {"pingInterval", std::string(instanceServer["pingInterval"].GetString())}, diff --git a/include/ccapi_cpp/service/ccapi_execution_management_service_mexc.h b/include/ccapi_cpp/service/ccapi_execution_management_service_mexc.h index 2daad48d..7f524816 100644 --- a/include/ccapi_cpp/service/ccapi_execution_management_service_mexc.h +++ b/include/ccapi_cpp/service/ccapi_execution_management_service_mexc.h @@ -360,7 +360,7 @@ class ExecutionManagementServiceMexc : public ExecutionManagementService { document.Parse(body.c_str()); std::string listenKey = document["listenKey"].GetString(); std::string url = that->baseUrlWs + "?listenKey=" + listenKey; - wsConnectionPtr->url = url; + wsConnectionPtr->setUrl(url); that->connect(wsConnectionPtr); that->extraPropertyByConnectionIdMap[wsConnectionPtr->id].insert({ {"listenKey", listenKey}, diff --git a/include/ccapi_cpp/service/ccapi_market_data_service.h b/include/ccapi_cpp/service/ccapi_market_data_service.h index 2ac851ea..cfefae6b 100644 --- a/include/ccapi_cpp/service/ccapi_market_data_service.h +++ b/include/ccapi_cpp/service/ccapi_market_data_service.h @@ -131,8 +131,7 @@ class MarketDataService : public Service { } std::shared_ptr>> streamPtr(nullptr); try { - streamPtr = that->createStream>>( - that->serviceContextPtr->ioContextPtr, that->serviceContextPtr->sslContextPtr, that->hostWs); + streamPtr = that->createWsStream(that->serviceContextPtr->ioContextPtr, that->serviceContextPtr->sslContextPtr); } catch (const beast::error_code& ec) { CCAPI_LOGGER_TRACE("fail"); std::vector correlationIdList; diff --git a/include/ccapi_cpp/service/ccapi_market_data_service_gemini.h b/include/ccapi_cpp/service/ccapi_market_data_service_gemini.h index 035f7612..432025a1 100644 --- a/include/ccapi_cpp/service/ccapi_market_data_service_gemini.h +++ b/include/ccapi_cpp/service/ccapi_market_data_service_gemini.h @@ -131,7 +131,7 @@ class MarketDataServiceGemini : public MarketDataService { if (marketDepthSubscribedToExchange == 1) { this->l2UpdateIsReplaceByConnectionIdChannelIdSymbolIdMap[wsConnectionPtr->id][channelId][symbolId] = true; } - auto exchangeSubscriptionId = wsConnectionPtr->url; + auto exchangeSubscriptionId = wsConnectionPtr->getUrl(); this->channelIdSymbolIdByConnectionIdExchangeSubscriptionIdMap[wsConnectionPtr->id][exchangeSubscriptionId][CCAPI_CHANNEL_ID] = channelId; this->channelIdSymbolIdByConnectionIdExchangeSubscriptionIdMap[wsConnectionPtr->id][exchangeSubscriptionId][CCAPI_SYMBOL_ID] = symbolId; std::vector correlationIdList_2 = @@ -225,7 +225,7 @@ class MarketDataServiceGemini : public MarketDataService { #ifndef CCAPI_USE_BOOST_BEAST_WEBSOCKET marketDataMessage.exchangeSubscriptionId = wsConnection.url; #else - marketDataMessage.exchangeSubscriptionId = wsConnectionPtr->url; + marketDataMessage.exchangeSubscriptionId = wsConnectionPtr.getUrl(); #endif TimePoint time = timeReceived; auto it = document.FindMember("timestampms"); diff --git a/include/ccapi_cpp/service/ccapi_market_data_service_kucoin_base.h b/include/ccapi_cpp/service/ccapi_market_data_service_kucoin_base.h index cc182604..e750e491 100644 --- a/include/ccapi_cpp/service/ccapi_market_data_service_kucoin_base.h +++ b/include/ccapi_cpp/service/ccapi_market_data_service_kucoin_base.h @@ -124,7 +124,7 @@ class MarketDataServiceKucoinBase : public MarketDataService { urlWebsocketBase += std::string(instanceServer["endpoint"].GetString()); urlWebsocketBase += "?token="; urlWebsocketBase += std::string(document["data"]["token"].GetString()); - wsConnectionPtr->url = urlWebsocketBase; + wsConnectionPtr->setUrl(urlWebsocketBase); that->connect(wsConnectionPtr); for (const auto& subscription : wsConnectionPtr->subscriptionList) { auto instrument = subscription.getInstrument(); diff --git a/include/ccapi_cpp/service/ccapi_service.h b/include/ccapi_cpp/service/ccapi_service.h index 7ce93e31..62bc7564 100644 --- a/include/ccapi_cpp/service/ccapi_service.h +++ b/include/ccapi_cpp/service/ccapi_service.h @@ -379,6 +379,8 @@ class Service : public std::enable_shared_from_this { std::shared_ptr httpConnectionPtr(new HttpConnection(host, port, streamPtr)); CCAPI_LOGGER_DEBUG("httpConnection = " + toString(*httpConnectionPtr)); std::shared_ptr newResolverPtr(new tcp::resolver(*this->serviceContextPtr->ioContextPtr)); + CCAPI_LOGGER_TRACE("host = " + host); + CCAPI_LOGGER_TRACE("port = " + port); newResolverPtr->async_resolve(host, port, beast::bind_front_handler(&Service::onResolve, shared_from_this(), httpConnectionPtr, newResolverPtr, req, errorHandler, responseHandler, timeoutMilliSeconds)); @@ -500,19 +502,10 @@ class Service : public std::enable_shared_from_this { } #ifndef CCAPI_USE_BOOST_BEAST_WEBSOCKET #else - template <> - std::shared_ptr>> createStream(std::shared_ptr iocPtr, - std::shared_ptr ctxPtr, - const std::string& host) { - CCAPI_LOGGER_TRACE("host = " + host) + std::shared_ptr>> createWsStream(std::shared_ptr iocPtr, + std::shared_ptr ctxPtr) { std::shared_ptr>> streamPtr( new beast::websocket::stream>(*iocPtr, *ctxPtr)); - // Set SNI Hostname (many hosts need this to handshake successfully) - if (!SSL_set_tlsext_host_name(streamPtr->next_layer().native_handle(), host.c_str())) { - beast::error_code ec{static_cast(::ERR_get_error()), net::error::get_ssl_category()}; - CCAPI_LOGGER_DEBUG("error SSL_set_tlsext_host_name: " + ec.message()); - throw ec; - } return streamPtr; } #endif @@ -1213,13 +1206,16 @@ class Service : public std::enable_shared_from_this { WsConnection& wsConnection = *wsConnectionPtr; wsConnection.status = WsConnection::Status::CONNECTING; CCAPI_LOGGER_DEBUG("connection initialization on id " + wsConnection.id); - std::string url = wsConnection.url; + std::string url = wsConnection.getUrl(); CCAPI_LOGGER_DEBUG("url = " + url); this->startResolveWs(wsConnectionPtr); CCAPI_LOGGER_FUNCTION_EXIT; } void startResolveWs(std::shared_ptr wsConnectionPtr) { std::shared_ptr newResolverPtr(new tcp::resolver(*this->serviceContextPtr->ioContextPtr)); + CCAPI_LOGGER_TRACE("wsConnectionPtr = " + wsConnectionPtr->toString()); + CCAPI_LOGGER_TRACE("wsConnectionPtr->host = " + wsConnectionPtr->host); + CCAPI_LOGGER_TRACE("wsConnectionPtr->port = " + wsConnectionPtr->port); newResolverPtr->async_resolve(wsConnectionPtr->host, wsConnectionPtr->port, beast::bind_front_handler(&Service::onResolveWs, shared_from_this(), wsConnectionPtr, newResolverPtr)); } @@ -1237,6 +1233,14 @@ class Service : public std::enable_shared_from_this { if (timeoutMilliSeconds > 0) { beast::get_lowest_layer(stream).expires_after(std::chrono::milliseconds(timeoutMilliSeconds)); } + // Set SNI Hostname (many hosts need this to handshake successfully) + CCAPI_LOGGER_TRACE("wsConnectionPtr->host = " + wsConnectionPtr->host) + if (!SSL_set_tlsext_host_name(stream.next_layer().native_handle(), wsConnectionPtr->host.c_str())) { + beast::error_code ec{static_cast(::ERR_get_error()), net::error::get_ssl_category()}; + CCAPI_LOGGER_DEBUG("error SSL_set_tlsext_host_name: " + ec.message()); + this->onError(Event::Type::SUBSCRIPTION_STATUS, Message::Type::SUBSCRIPTION_FAILURE, ec, "set SNI Hostname", wsConnectionPtr->correlationIdList); + return; + } CCAPI_LOGGER_TRACE("before async_connect"); beast::get_lowest_layer(stream).async_connect(tcpResolverResults, beast::bind_front_handler(&Service::onConnectWs, shared_from_this(), wsConnectionPtr)); CCAPI_LOGGER_TRACE("after async_connect"); @@ -1250,7 +1254,8 @@ class Service : public std::enable_shared_from_this { } CCAPI_LOGGER_TRACE("connected"); CCAPI_LOGGER_TRACE("ep.port() = " + std::to_string(ep.port())); - wsConnectionPtr->hostHttpHeaderValue = this->hostWs + ':' + std::to_string(ep.port()); + wsConnectionPtr->hostHttpHeaderValue = wsConnectionPtr->host + ':' + std::to_string(ep.port()); + CCAPI_LOGGER_TRACE("wsConnectionPtr->hostHttpHeaderValue = " + wsConnectionPtr->hostHttpHeaderValue); beast::websocket::stream>& stream = *wsConnectionPtr->streamPtr; beast::get_lowest_layer(stream).socket().set_option(tcp::no_delay(true)); CCAPI_LOGGER_TRACE("before ssl async_handshake"); @@ -1348,7 +1353,7 @@ class Service : public std::enable_shared_from_this { WsConnection& wsConnection = *wsConnectionPtr; wsConnection.status = WsConnection::Status::OPEN; CCAPI_LOGGER_INFO("connection " + toString(wsConnection) + " established"); - auto urlBase = UtilString::split(wsConnection.url, "?").at(0); + auto urlBase = UtilString::split(wsConnection.getUrl(), "?").at(0); this->connectNumRetryOnFailByConnectionUrlMap[urlBase] = 0; Event event; event.setType(Event::Type::SESSION_STATUS); @@ -1360,7 +1365,7 @@ class Service : public std::enable_shared_from_this { message.setCorrelationIdList(correlationIdList); Element element; element.insert(CCAPI_CONNECTION_ID, wsConnection.id); - element.insert(CCAPI_CONNECTION_URL, wsConnection.url); + element.insert(CCAPI_CONNECTION_URL, wsConnection.getUrl()); message.setElementList({element}); event.setMessageList({message}); this->eventHandler(event, nullptr); @@ -1427,7 +1432,7 @@ class Service : public std::enable_shared_from_this { this->onError(Event::Type::SUBSCRIPTION_STATUS, Message::Type::SUBSCRIPTION_FAILURE, "connection " + toString(wsConnection) + " has failed before opening"); WsConnection thisWsConnection = wsConnection; this->wsConnectionByIdMap.erase(thisWsConnection.id); - auto urlBase = UtilString::split(thisWsConnection.url, "?").at(0); + auto urlBase = UtilString::split(thisWsConnection.getUrl(), "?").at(0); long seconds = std::round(UtilAlgorithm::exponentialBackoff(1, 1, 2, std::min(this->connectNumRetryOnFailByConnectionUrlMap[urlBase], 6))); CCAPI_LOGGER_INFO("about to set timer for " + toString(seconds) + " seconds"); if (this->connectRetryOnFailTimerByConnectionIdMap.find(thisWsConnection.id) != this->connectRetryOnFailTimerByConnectionIdMap.end()) { @@ -1460,8 +1465,7 @@ class Service : public std::enable_shared_from_this { std::shared_ptr thatWsConnectionPtr = wsConnectionPtr; std::shared_ptr>> streamPtr(nullptr); try { - streamPtr = this->createStream>>(this->serviceContextPtr->ioContextPtr, - this->serviceContextPtr->sslContextPtr, this->hostWs); + streamPtr = this->createWsStream(this->serviceContextPtr->ioContextPtr, this->serviceContextPtr->sslContextPtr); } catch (const beast::error_code& ec) { throw ec; } @@ -1520,7 +1524,7 @@ class Service : public std::enable_shared_from_this { message.setType(Message::Type::SESSION_CONNECTION_DOWN); Element element; element.insert(CCAPI_CONNECTION_ID, wsConnection.id); - element.insert(CCAPI_CONNECTION_URL, wsConnection.url); + element.insert(CCAPI_CONNECTION_URL, wsConnection.getUrl()); element.insert(CCAPI_REASON, reason); message.setElementList({element}); std::vector correlationIdList; diff --git a/performance/CMakeLists.txt b/performance/CMakeLists.txt index 592c5639..848d4601 100644 --- a/performance/CMakeLists.txt +++ b/performance/CMakeLists.txt @@ -39,4 +39,5 @@ if (WIN32) set(ADDITIONAL_LINK_LIBRARIES ws2_32) endif() link_libraries(OpenSSL::Crypto OpenSSL::SSL ${ADDITIONAL_LINK_LIBRARIES}) +add_compile_options(-Wno-deprecated) add_subdirectory(src/rest_vs_fix) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 2b355fd2..e7cd7ea1 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -31,6 +31,7 @@ if (WIN32) endif() link_libraries(OpenSSL::Crypto OpenSSL::SSL ${ADDITIONAL_LINK_LIBRARIES}) set(SOURCE_LOGGER ${CCAPI_PROJECT_DIR}/test/ccapi_logger.cpp) +add_compile_options(-Wno-deprecated) if(BUILD_TEST_BUILD) add_subdirectory(test_build) endif()