diff --git a/CHANGELOG.md b/CHANGELOG.md index 73944a7..46fc213 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,17 @@ # Changelog +## 0.35.0 - 2025-05-13 + +### Enhancements +- Added a `v3::StatMsg` record with an expanded 64-bit `quantity` field +- Added `kDbnVersion` constants to each version namespace: `v1`, `v2`, and `v3` +- Added `kUndefStatQuantity` constants to each version namespace +- Added new off-market publishers for Eurex, and European Energy Exchange (EEX) +- Increased live subscription symbol chunking size + ## 0.34.2 - 2025-05-06 -#### Bug fixes +### Bug fixes - Fixed potential for unaligned records in live and historical streaming requests ## 0.34.1 - 2025-04-29 diff --git a/CMakeLists.txt b/CMakeLists.txt index db048cc..2927ce1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,7 @@ cmake_minimum_required(VERSION 3.24..4.0) project( databento - VERSION 0.34.2 + VERSION 0.35.0 LANGUAGES CXX DESCRIPTION "Official Databento client library" ) diff --git a/include/databento/publishers.hpp b/include/databento/publishers.hpp index 7aa9c31..baafcf4 100644 --- a/include/databento/publishers.hpp +++ b/include/databento/publishers.hpp @@ -31,7 +31,7 @@ enum class Venue : std::uint16_t { Xase = 11, // NYSE Arca Arcx = 12, - // NYSE Chicago, Inc. + // NYSE Texas, Inc. Xchi = 13, // Investors Exchange Iexg = 14, @@ -47,21 +47,21 @@ enum class Venue : std::uint16_t { Eprl = 19, // NYSE American Options Amxo = 20, - // BOX Options Exchange + // BOX Options Xbox = 21, - // Cboe Options Exchange + // Cboe Options Xcbo = 22, // MIAX Emerald Emld = 23, - // Cboe EDGX Options Exchange + // Cboe EDGX Options Edgo = 24, - // ISE Gemini Exchange + // Nasdaq GEMX Gmni = 25, - // International Securities Exchange, LLC + // Nasdaq ISE Xisx = 26, - // ISE Mercury, LLC + // Nasdaq MRX Mcry = 27, - // Miami International Securities Exchange + // MIAX Options Xmio = 28, // NYSE Arca Options Arco = 29, @@ -69,17 +69,17 @@ enum class Venue : std::uint16_t { Opra = 30, // MIAX Pearl Mprl = 31, - // Nasdaq Options Market + // Nasdaq Options Xndq = 32, - // Nasdaq OMX BX Options + // Nasdaq BX Options Xbxo = 33, - // Cboe C2 Options Exchange + // Cboe C2 Options C2Ox = 34, - // Nasdaq OMX PHLX + // Nasdaq PHLX Xphl = 35, - // Cboe BZX Options Exchange + // Cboe BZX Options Bato = 36, - // MEMX LLC Options + // MEMX Options Mxop = 37, // ICE Futures Europe (Commodities) Ifeu = 38, @@ -135,7 +135,7 @@ enum class Dataset : std::uint16_t { XcisPillar = 10, // NYSE American Integrated XasePillar = 11, - // NYSE Chicago Integrated + // NYSE Texas Integrated XchiPillar = 12, // NYSE National BBO XcisBbo = 13, @@ -217,7 +217,7 @@ enum class Publisher : std::uint16_t { XcisPillarXcis = 10, // NYSE American Integrated XasePillarXase = 11, - // NYSE Chicago Integrated + // NYSE Texas Integrated XchiPillarXchi = 12, // NYSE National BBO XcisBboXcis = 13, @@ -233,15 +233,15 @@ enum class Publisher : std::uint16_t { XnasNlsFinc = 18, // FINRA/NYSE TRF XnysTradesFiny = 19, - // OPRA - NYSE American + // OPRA - NYSE American Options OpraPillarAmxo = 20, - // OPRA - Boston Options Exchange + // OPRA - BOX Options OpraPillarXbox = 21, - // OPRA - Cboe Options Exchange + // OPRA - Cboe Options OpraPillarXcbo = 22, // OPRA - MIAX Emerald OpraPillarEmld = 23, - // OPRA - Cboe EDGX Options Exchange + // OPRA - Cboe EDGX Options OpraPillarEdgo = 24, // OPRA - Nasdaq GEMX OpraPillarGmni = 25, @@ -249,29 +249,29 @@ enum class Publisher : std::uint16_t { OpraPillarXisx = 26, // OPRA - Nasdaq MRX OpraPillarMcry = 27, - // OPRA - Miami International Securities + // OPRA - MIAX Options OpraPillarXmio = 28, - // OPRA - NYSE Arca + // OPRA - NYSE Arca Options OpraPillarArco = 29, // OPRA - Options Price Reporting Authority OpraPillarOpra = 30, // OPRA - MIAX Pearl OpraPillarMprl = 31, - // OPRA - Nasdaq Options Market + // OPRA - Nasdaq Options OpraPillarXndq = 32, // OPRA - Nasdaq BX Options OpraPillarXbxo = 33, - // OPRA - Cboe C2 Options Exchange + // OPRA - Cboe C2 Options OpraPillarC2Ox = 34, // OPRA - Nasdaq PHLX OpraPillarXphl = 35, // OPRA - Cboe BZX Options OpraPillarBato = 36, - // OPRA - MEMX Options Exchange + // OPRA - MEMX Options OpraPillarMxop = 37, // IEX TOPS IexgTopsIexg = 38, - // DBEQ Basic - NYSE Chicago + // DBEQ Basic - NYSE Texas DbeqBasicXchi = 39, // DBEQ Basic - NYSE National DbeqBasicXcis = 40, @@ -289,7 +289,7 @@ enum class Publisher : std::uint16_t { XnasQbboXnas = 46, // Nasdaq Trades XnasNlsXnas = 47, - // Databento US Equities Plus - NYSE Chicago + // Databento US Equities Plus - NYSE Texas EqusPlusXchi = 48, // Databento US Equities Plus - NYSE National EqusPlusXcis = 49, @@ -317,7 +317,7 @@ enum class Publisher : std::uint16_t { EqusPlusEqus = 60, // OPRA - MIAX Sapphire OpraPillarSphr = 61, - // Databento US Equities (All Feeds) - NYSE Chicago + // Databento US Equities (All Feeds) - NYSE Texas EqusAllXchi = 62, // Databento US Equities (All Feeds) - NYSE National EqusAllXcis = 63, @@ -399,6 +399,10 @@ enum class Publisher : std::uint16_t { XeurEobiXeur = 101, // European Energy Exchange EOBI XeerEobiXeer = 102, + // Eurex EOBI - Off-Market Trades + XeurEobiXoff = 103, + // European Energy Exchange EOBI - Off-Market Trades + XeerEobiXoff = 104, }; // Get a Publisher's Venue. diff --git a/include/databento/record.hpp b/include/databento/record.hpp index c238ee3..5f8f55c 100644 --- a/include/databento/record.hpp +++ b/include/databento/record.hpp @@ -20,7 +20,8 @@ namespace databento { // Forward declare namespace v3 { struct InstrumentDefMsg; -} +struct StatMsg; +} // namespace v3 // Common data for all Databento Records. struct RecordHeader { @@ -501,6 +502,7 @@ static_assert(alignof(ImbalanceMsg) == 8, "Must have 8-byte alignment"); struct StatMsg { static bool HasRType(RType rtype) { return rtype == RType::Statistics; } + v3::StatMsg ToV3() const; UnixNanos IndexTs() const { return ts_recv; } RecordHeader hd; diff --git a/include/databento/v1.hpp b/include/databento/v1.hpp index cb206ae..cf2b98a 100644 --- a/include/databento/v1.hpp +++ b/include/databento/v1.hpp @@ -1,7 +1,7 @@ #pragma once -#include "databento/constants.hpp" // kSymbolCstrLen -#include "databento/datetime.hpp" // UnixNanos +#include "databento/constants.hpp" // kAssetCstrLen, kSymbolCstrLen, kUndefStatQuantity +#include "databento/datetime.hpp" // UnixNanos #include "databento/enums.hpp" #include "databento/record.hpp" #include "databento/v2.hpp" @@ -13,8 +13,11 @@ struct InstrumentDefMsg; } namespace v1 { +static constexpr std::uint8_t kDbnVersion = 1; static constexpr std::size_t kSymbolCstrLen = 22; static constexpr std::size_t kAssetCstrLen = databento::kAssetCstrLen; +static constexpr std::int32_t kUndefStatQuantity = + databento::kUndefStatQuantity; using MboMsg = databento::MboMsg; using TradeMsg = databento::TradeMsg; diff --git a/include/databento/v2.hpp b/include/databento/v2.hpp index e64ccd4..b8adabc 100644 --- a/include/databento/v2.hpp +++ b/include/databento/v2.hpp @@ -1,11 +1,14 @@ #pragma once -#include "databento/constants.hpp" // kSymbolCstrLen +#include "databento/constants.hpp" // kAssetCstrLen, kSymbolCstrLen, kUndefStatQuantity #include "databento/record.hpp" namespace databento::v2 { +static constexpr std::uint8_t kDbnVersion = 2; static constexpr std::size_t kSymbolCstrLen = databento::kSymbolCstrLen; static constexpr std::size_t kAssetCstrLen = databento::kAssetCstrLen; +static constexpr std::int32_t kUndefStatQuantity = + databento::kUndefStatQuantity; using MboMsg = databento::MboMsg; using TradeMsg = databento::TradeMsg; diff --git a/include/databento/v3.hpp b/include/databento/v3.hpp index 5472712..d7ac98c 100644 --- a/include/databento/v3.hpp +++ b/include/databento/v3.hpp @@ -2,6 +2,7 @@ #include #include +#include #include "databento/constants.hpp" // kSymbolCstrLen #include "databento/datetime.hpp" // UnixNanos @@ -9,8 +10,11 @@ #include "databento/record.hpp" // RecordHeader namespace databento::v3 { +static constexpr std::uint8_t kDbnVersion = 3; static constexpr std::size_t kSymbolCstrLen = databento::kSymbolCstrLen; static constexpr std::size_t kAssetCstrLen = 11; +static constexpr std::int64_t kUndefStatQuantity = + std::numeric_limits::max(); using MboMsg = databento::MboMsg; using TradeMsg = databento::TradeMsg; @@ -28,7 +32,6 @@ using Cbbo1MMsg = databento::Cbbo1MMsg; using OhlcvMsg = databento::OhlcvMsg; using StatusMsg = databento::StatusMsg; using ImbalanceMsg = databento::ImbalanceMsg; -using StatMsg = databento::StatMsg; using ErrorMsg = databento::ErrorMsg; using SymbolMappingMsg = databento::SymbolMappingMsg; using SystemMsg = databento::SystemMsg; @@ -133,13 +136,50 @@ static_assert(alignof(InstrumentDefMsg) == 8, "Must have 8-byte alignment"); static_assert(kMaxRecordLen == sizeof(InstrumentDefMsg) + sizeof(UnixNanos), "v3 definition with ts_out should be the largest record"); +/// A statistics message. A catchall for various data disseminated by +/// publishers. The `stat_type` indicates the statistic contained in the +/// message. +struct StatMsg { + static bool HasRType(RType rtype) { return rtype == RType::Statistics; } + + UnixNanos IndexTs() const { return ts_recv; } + + RecordHeader hd; + UnixNanos ts_recv; + UnixNanos ts_ref; + std::int64_t price; + std::int64_t quantity; + std::uint32_t sequence; + TimeDeltaNanos ts_in_delta; + StatType stat_type; + std::uint16_t channel_id; + StatUpdateAction update_action; + std::uint8_t stat_flags; + std::array reserved; +}; +static_assert(sizeof(StatMsg) == 80, "StatMsg size must match Rust"); +static_assert(alignof(StatMsg) == 8, "Must have 8-byte alignment"); + bool operator==(const InstrumentDefMsg& lhs, const InstrumentDefMsg& rhs); inline bool operator!=(const InstrumentDefMsg& lhs, const InstrumentDefMsg& rhs) { return !(lhs == rhs); } +inline bool operator==(const StatMsg& lhs, const StatMsg& rhs) { + return std::tie(lhs.hd, lhs.ts_recv, lhs.ts_ref, lhs.price, lhs.quantity, + lhs.sequence, lhs.ts_in_delta, lhs.stat_type, lhs.channel_id, + lhs.update_action, lhs.stat_flags) == + std::tie(rhs.hd, rhs.ts_recv, rhs.ts_ref, rhs.price, rhs.quantity, + rhs.sequence, rhs.ts_in_delta, rhs.stat_type, rhs.channel_id, + rhs.update_action, rhs.stat_flags); +} +inline bool operator!=(const StatMsg& lhs, const StatMsg& rhs) { + return !(lhs == rhs); +} std::string ToString(const InstrumentDefMsg& instr_def_msg); std::ostream& operator<<(std::ostream& stream, const InstrumentDefMsg& instr_def_msg); +std::string ToString(const StatMsg& stat_msg); +std::ostream& operator<<(std::ostream& stream, const StatMsg& stat_msg); } // namespace databento::v3 diff --git a/pkg/PKGBUILD b/pkg/PKGBUILD index 38dd21c..7827102 100644 --- a/pkg/PKGBUILD +++ b/pkg/PKGBUILD @@ -1,7 +1,7 @@ # Maintainer: Databento _pkgname=databento-cpp pkgname=databento-cpp-git -pkgver=0.34.2 +pkgver=0.35.0 pkgrel=1 pkgdesc="Official C++ client for Databento" arch=('any') diff --git a/scripts/get_version.sh b/scripts/get_version.sh index f9db315..b75d052 100755 --- a/scripts/get_version.sh +++ b/scripts/get_version.sh @@ -2,4 +2,6 @@ SCRIPTS_DIR="$(cd "$(dirname "$0")" || exit; pwd -P)" PROJECT_ROOT_DIR="$(dirname "${SCRIPTS_DIR}")" -grep '^project("databento" VERSION' "${PROJECT_ROOT_DIR}/CMakeLists.txt" | cut -d' ' -f3 +grep --after-context=2 '^project($' "${PROJECT_ROOT_DIR}/CMakeLists.txt" \ + | grep --perl-regexp --only-matching 'VERSION \d+\.\d+\.\d+' \ + | cut --delimiter=' ' --fields=2 diff --git a/src/live_blocking.cpp b/src/live_blocking.cpp index d36172c..1fdeb3b 100644 --- a/src/live_blocking.cpp +++ b/src/live_blocking.cpp @@ -113,7 +113,7 @@ void LiveBlocking::Subscribe(std::string_view sub_msg, const std::vector& symbols, bool use_snapshot) { static constexpr auto kMethodName = "Live::Subscribe"; - constexpr std::ptrdiff_t kSymbolMaxChunkSize = 128; + constexpr std::ptrdiff_t kSymbolMaxChunkSize = 500; if (symbols.empty()) { throw InvalidArgumentError{kMethodName, "symbols", @@ -121,8 +121,8 @@ void LiveBlocking::Subscribe(std::string_view sub_msg, } auto symbols_it = symbols.begin(); while (symbols_it != symbols.end()) { - const auto chunk_size = - std::min(kSymbolMaxChunkSize, std::distance(symbols_it, symbols.end())); + const auto distance_from_end = std::distance(symbols_it, symbols.end()); + const auto chunk_size = std::min(kSymbolMaxChunkSize, distance_from_end); std::ostringstream chunked_sub_msg; chunked_sub_msg << sub_msg << "|symbols=" diff --git a/src/publishers.cpp b/src/publishers.cpp index cd90ccd..a606f25 100644 --- a/src/publishers.cpp +++ b/src/publishers.cpp @@ -895,6 +895,12 @@ Venue PublisherVenue(Publisher publisher) { case Publisher::XeerEobiXeer: { return Venue::Xeer; } + case Publisher::XeurEobiXoff: { + return Venue::Xoff; + } + case Publisher::XeerEobiXoff: { + return Venue::Xoff; + } default: { throw InvalidArgumentError{ "PublisherVenue", "publisher", @@ -1211,6 +1217,12 @@ Dataset PublisherDataset(Publisher publisher) { case Publisher::XeerEobiXeer: { return Dataset::XeerEobi; } + case Publisher::XeurEobiXoff: { + return Dataset::XeurEobi; + } + case Publisher::XeerEobiXoff: { + return Dataset::XeerEobi; + } default: { throw InvalidArgumentError{ "PublisherDataset", "publisher", @@ -1528,6 +1540,12 @@ const char* ToString(Publisher publisher) { case Publisher::XeerEobiXeer: { return "XEER.EOBI.XEER"; } + case Publisher::XeurEobiXoff: { + return "XEUR.EOBI.XOFF"; + } + case Publisher::XeerEobiXoff: { + return "XEER.EOBI.XOFF"; + } default: { return "Unknown"; } @@ -1847,6 +1865,12 @@ Publisher FromString(const std::string& str) { if (str == "XEER.EOBI.XEER") { return Publisher::XeerEobiXeer; } + if (str == "XEUR.EOBI.XOFF") { + return Publisher::XeurEobiXoff; + } + if (str == "XEER.EOBI.XOFF") { + return Publisher::XeerEobiXoff; + } throw InvalidArgumentError{"FromString", "str", "unknown value '" + str + '\''}; } diff --git a/src/record.cpp b/src/record.cpp index c2c6b0a..5212f86 100644 --- a/src/record.cpp +++ b/src/record.cpp @@ -2,6 +2,7 @@ #include +#include "databento/constants.hpp" #include "databento/enums.hpp" #include "databento/exceptions.hpp" // InvalidArgumentError #include "databento/fixed_price.hpp" @@ -200,6 +201,26 @@ databento::v3::InstrumentDefMsg InstrumentDefMsg::ToV3() const { return ret; } +using databento::StatMsg; + +databento::v3::StatMsg StatMsg::ToV3() const { + return databento::v3::StatMsg{ + RecordHeader{sizeof(v3::StatMsg) / RecordHeader::kLengthMultiplier, + RType::Statistics, hd.publisher_id, hd.instrument_id, + hd.ts_event}, + ts_recv, + ts_ref, + price, + quantity == kUndefStatQuantity ? v3::kUndefStatQuantity : quantity, + sequence, + ts_in_delta, + stat_type, + channel_id, + update_action, + stat_flags, + {}}; +} + bool databento::operator==(const InstrumentDefMsg& lhs, const InstrumentDefMsg& rhs) { return lhs.hd == rhs.hd && lhs.ts_recv == rhs.ts_recv && diff --git a/src/v3.cpp b/src/v3.cpp index 5e69702..a021f43 100644 --- a/src/v3.cpp +++ b/src/v3.cpp @@ -157,4 +157,24 @@ std::ostream& operator<<(std::ostream& stream, .AddField("leg_side", instr_def_msg.leg_side) .Finish(); } + +std::string ToString(const StatMsg& stat_msg) { return MakeString(stat_msg); } +std::ostream& operator<<(std::ostream& stream, const StatMsg& stat_msg) { + return StreamOpBuilder{stream} + .SetSpacer("\n ") + .SetTypeName("StatMsg") + .Build() + .AddField("hd", stat_msg.hd) + .AddField("ts_recv", stat_msg.ts_recv) + .AddField("ts_ref", stat_msg.ts_ref) + .AddField("price", FixPx{stat_msg.price}) + .AddField("quantity", stat_msg.quantity) + .AddField("sequence", stat_msg.sequence) + .AddField("ts_in_delta", stat_msg.ts_in_delta) + .AddField("stat_type", stat_msg.stat_type) + .AddField("channel_id", stat_msg.channel_id) + .AddField("update_action", stat_msg.update_action) + .AddField("stat_flags", stat_msg.stat_flags) + .Finish(); +} } // namespace databento::v3 diff --git a/tests/src/live_blocking_tests.cpp b/tests/src/live_blocking_tests.cpp index 70dc572..91517e6 100644 --- a/tests/src/live_blocking_tests.cpp +++ b/tests/src/live_blocking_tests.cpp @@ -107,7 +107,7 @@ TEST_F(LiveBlockingTests, TestSubscriptionChunkingUnixNanos) { constexpr auto kTsOut = false; constexpr auto kDataset = dataset::kXnasItch; const auto kSymbol = "TEST"; - const std::size_t kSymbolCount = 1000; + const std::size_t kSymbolCount = 1001; const auto kSchema = Schema::Ohlcv1M; const auto kSType = SType::RawSymbol; @@ -117,9 +117,9 @@ TEST_F(LiveBlockingTests, TestSubscriptionChunkingUnixNanos) { self.Accept(); self.Authenticate(); std::size_t i{}; - while (i < 1000) { + while (i < kSymbolCount) { const auto chunk_size = - std::min(static_cast(128), kSymbolCount - i); + std::min(static_cast(500), kSymbolCount - i); const std::vector symbols_chunk(chunk_size, kSymbol); self.Subscribe(symbols_chunk, kSchema, kSType); i += chunk_size; @@ -162,7 +162,7 @@ TEST_F(LiveBlockingTests, TestSubscriptionChunkingStringStart) { constexpr auto kTsOut = false; constexpr auto kDataset = dataset::kXnasItch; const auto kSymbol = "TEST"; - const std::size_t kSymbolCount = 1000; + const std::size_t kSymbolCount = 1001; const auto kSchema = Schema::Ohlcv1M; const auto kSType = SType::RawSymbol; const auto kStart = "2020-01-01T00:00:00"; @@ -174,9 +174,9 @@ TEST_F(LiveBlockingTests, TestSubscriptionChunkingStringStart) { self.Accept(); self.Authenticate(); std::size_t i{}; - while (i < 1000) { + while (i < kSymbolCount) { const auto chunk_size = - std::min(static_cast(128), kSymbolCount - i); + std::min(static_cast(500), kSymbolCount - i); const std::vector symbols_chunk(chunk_size, kSymbol); self.Subscribe(symbols_chunk, kSchema, kSType, kStart); i += chunk_size; @@ -195,7 +195,7 @@ TEST_F(LiveBlockingTests, TestSubscribeSnapshot) { constexpr auto kTsOut = false; constexpr auto kDataset = dataset::kXnasItch; const auto kSymbol = "TEST"; - const std::size_t kSymbolCount = 1000; + const std::size_t kSymbolCount = 1001; const auto kSchema = Schema::Ohlcv1M; const auto kSType = SType::RawSymbol; const auto kUseSnapshot = true; @@ -207,9 +207,9 @@ TEST_F(LiveBlockingTests, TestSubscribeSnapshot) { self.Accept(); self.Authenticate(); std::size_t i{}; - while (i < 1000) { + while (i < kSymbolCount) { const auto chunk_size = - std::min(static_cast(128), kSymbolCount - i); + std::min(static_cast(500), kSymbolCount - i); const std::vector symbols_chunk(chunk_size, kSymbol); self.SubscribeWithSnapshot(symbols_chunk, kSchema, kSType); i += chunk_size; diff --git a/tests/src/mock_lsg_server.cpp b/tests/src/mock_lsg_server.cpp index c497769..84a9aa7 100644 --- a/tests/src/mock_lsg_server.cpp +++ b/tests/src/mock_lsg_server.cpp @@ -54,7 +54,7 @@ void MockLsgServer::Accept() { } std::string MockLsgServer::Receive() { - std::string received(1024, 0); + std::string received(static_cast(32 * 1024), 0); char c{}; std::size_t read_size{}; // Read char by char until newline @@ -68,8 +68,8 @@ std::string MockLsgServer::Receive() { } c = received[read_size]; ++read_size; - } while (c != '\n' && read_size < 1024); - if (read_size == 1024) { + } while (c != '\n' && read_size < received.size()); + if (read_size == received.size()) { throw TcpError{{}, "Overran buffer in MockLsgServer"}; } received.resize(read_size);