From a2ddcccdddb6e182aa4678bd8416aaa6a65c3270 Mon Sep 17 00:00:00 2001 From: Andrey Serebryanskiy Date: Wed, 27 Aug 2025 14:26:25 +0000 Subject: [PATCH 01/25] [Kafka API] Fix bug in kafka transactions (#21310) --- .github/last_commit.txt | 2 +- src/api/protos/draft/persqueue_error_codes.proto | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index d9486b7e28..020246643c 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -aa02b3d7cea3b9461fe806646f99a51df36d695f +dda968ffaa0f48643ff8e16d4fe956e6aab18387 diff --git a/src/api/protos/draft/persqueue_error_codes.proto b/src/api/protos/draft/persqueue_error_codes.proto index 9a32a6f00b..7296cd932f 100644 --- a/src/api/protos/draft/persqueue_error_codes.proto +++ b/src/api/protos/draft/persqueue_error_codes.proto @@ -54,7 +54,8 @@ enum EErrorCode { KAFKA_INVALID_PRODUCER_EPOCH = 32; KAFKA_OUT_OF_ORDER_SEQUENCE_NUMBER = 33; - KAFKA_DUPLICATE_SEQUENCE_NUMBER = 34; + KAFKA_TRANSACTION_MISSING_SUPPORTIVE_PARTITION = 34; + KAFKA_DUPLICATE_SEQUENCE_NUMBER = 35; ERROR = 100; } From 95e9aa0a93c252a579121d6d5809c013d8fdf495 Mon Sep 17 00:00:00 2001 From: Andrey Zaspa Date: Wed, 27 Aug 2025 14:26:38 +0000 Subject: [PATCH 02/25] Added a new sdk method DescribeSystemView and a corresponded Grpc service (#20603) --- .github/last_commit.txt | 2 +- include/ydb-cpp-sdk/client/proto/accessor.h | 1 + include/ydb-cpp-sdk/client/table/table.h | 48 ++++++++ src/api/grpc/ydb_table_v1.proto | 3 + src/api/protos/ydb_table.proto | 26 ++++ src/client/table/impl/table_client.cpp | 30 +++++ src/client/table/impl/table_client.h | 1 + src/client/table/proto_accessor.cpp | 4 + src/client/table/table.cpp | 124 ++++++++++++++++++++ 9 files changed, 238 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 020246643c..a3cbe7e8a0 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -dda968ffaa0f48643ff8e16d4fe956e6aab18387 +2b8366c9a860fe12b5e3aa62a1585e1d0a1a2686 diff --git a/include/ydb-cpp-sdk/client/proto/accessor.h b/include/ydb-cpp-sdk/client/proto/accessor.h index bfa6f02e7b..b13dd6a1db 100644 --- a/include/ydb-cpp-sdk/client/proto/accessor.h +++ b/include/ydb-cpp-sdk/client/proto/accessor.h @@ -45,6 +45,7 @@ class TProtoAccessor { static const Ydb::Table::DescribeTableResult& GetProto(const NTable::TTableDescription& tableDescription); static const Ydb::Table::DescribeExternalDataSourceResult& GetProto(const NTable::TExternalDataSourceDescription&); static const Ydb::Table::DescribeExternalTableResult& GetProto(const NTable::TExternalTableDescription&); + static const Ydb::Table::DescribeSystemViewResult& GetProto(const NTable::TSystemViewDescription&); static const Ydb::Topic::DescribeTopicResult& GetProto(const NYdb::NTopic::TTopicDescription& topicDescription); static const Ydb::Topic::DescribeConsumerResult& GetProto(const NYdb::NTopic::TConsumerDescription& consumerDescription); static const Ydb::Monitoring::SelfCheckResult& GetProto(const NYdb::NMonitoring::TSelfCheckResult& selfCheckResult); diff --git a/include/ydb-cpp-sdk/client/table/table.h b/include/ydb-cpp-sdk/client/table/table.h index 280c28d9da..f85d62c76f 100644 --- a/include/ydb-cpp-sdk/client/table/table.h +++ b/include/ydb-cpp-sdk/client/table/table.h @@ -25,6 +25,7 @@ class Changefeed; class ChangefeedDescription; class DescribeExternalDataSourceResult; class DescribeExternalTableResult; +class DescribeSystemViewResult; class DescribeTableResult; class ExplicitPartitions; class GlobalIndexSettings; @@ -1082,6 +1083,7 @@ class TRenameItem { class TDescribeExternalDataSourceResult; class TDescribeExternalTableResult; +class TDescribeSystemViewResult; using TAsyncCreateSessionResult = NThreading::TFuture; using TAsyncDataQueryResult = NThreading::TFuture; @@ -1090,6 +1092,7 @@ using TAsyncExplainDataQueryResult = NThreading::TFuture; using TAsyncDescribeTableResult = NThreading::TFuture; using TAsyncDescribeExternalDataSourceResult = NThreading::TFuture; using TAsyncDescribeExternalTableResult = NThreading::TFuture; +using TAsyncDescribeSystemViewResult = NThreading::TFuture; using TAsyncBeginTransactionResult = NThreading::TFuture; using TAsyncCommitTransactionResult = NThreading::TFuture; using TAsyncTablePartIterator = NThreading::TFuture; @@ -1718,6 +1721,8 @@ struct TDescribeExternalDataSourceSettings : public TOperationRequestSettings {}; +struct TDescribeSystemViewSettings : public TOperationRequestSettings {}; + struct TExplainDataQuerySettings : public TOperationRequestSettings { FLUENT_SETTING_DEFAULT(bool, WithCollectFullDiagnostics, false); }; @@ -1806,6 +1811,9 @@ class TSession { TAsyncDescribeExternalTableResult DescribeExternalTable(const std::string& path, const TDescribeExternalTableSettings& settings = {}); + TAsyncDescribeSystemViewResult DescribeSystemView(const std::string& path, + const TDescribeSystemViewSettings& settings = {}); + TAsyncBeginTransactionResult BeginTransaction(const TTxSettings& txSettings = TTxSettings(), const TBeginTxSettings& settings = TBeginTxSettings()); @@ -2287,5 +2295,45 @@ class TDescribeExternalTableResult : public NScheme::TDescribePathResult { TExternalTableDescription ExternalTableDescription_; }; +//! Represents system view description +class TSystemViewDescription { +public: + TSystemViewDescription(Ydb::Table::DescribeSystemViewResult&& desc); + + // System view id + uint64_t GetSysViewId() const; + const std::string& GetSysViewName() const; + + // Columns info + const std::vector& GetPrimaryKeyColumns() const; + std::vector GetTableColumns() const; + + // Attributes + const std::unordered_map& GetAttributes() const; + +private: + TSystemViewDescription(); + + friend class NYdb::TProtoAccessor; + const Ydb::Table::DescribeSystemViewResult& GetProto() const; + + class TImpl; + std::shared_ptr Impl_; +}; + +//! Represents the result of a DescribeSystemView call. +class TDescribeSystemViewResult : public NScheme::TDescribePathResult { +public: + TDescribeSystemViewResult( + TStatus&& status, + Ydb::Table::DescribeSystemViewResult&& desc + ); + + TSystemViewDescription GetSystemViewDescription() const; + +private: + TSystemViewDescription SystemViewDescription_; +}; + } // namespace NTable } // namespace NYdb diff --git a/src/api/grpc/ydb_table_v1.proto b/src/api/grpc/ydb_table_v1.proto index 66b508bdd3..28fb26c5fc 100644 --- a/src/api/grpc/ydb_table_v1.proto +++ b/src/api/grpc/ydb_table_v1.proto @@ -91,4 +91,7 @@ service TableService { // Returns information about a given external table. rpc DescribeExternalTable(Table.DescribeExternalTableRequest) returns (Table.DescribeExternalTableResponse); + + // Returns information about a given system view table. + rpc DescribeSystemView(Table.DescribeSystemViewRequest) returns (Table.DescribeSystemViewResponse); } diff --git a/src/api/protos/ydb_table.proto b/src/api/protos/ydb_table.proto index 2e6aaf102c..4c6bd864ec 100644 --- a/src/api/protos/ydb_table.proto +++ b/src/api/protos/ydb_table.proto @@ -1342,3 +1342,29 @@ message DescribeExternalTableResult { repeated ColumnMeta columns = 5; map content = 6; } + +// Returns information about a system view with a given path. +message DescribeSystemViewRequest { + Ydb.Operations.OperationParams operation_params = 1; + string path = 2; +} + +message DescribeSystemViewResponse { + // Holds DescribeSystemViewResult in case of a successful call. + Ydb.Operations.Operation operation = 1; +} + +message DescribeSystemViewResult { + // Description of scheme object + Ydb.Scheme.Entry self = 1; + // Id of system view (Enum value from NKikimrSysView.ESysViewType) + uint64 sys_view_id = 2; + // NameId of system view + string sys_view_name = 3; + // List of columns + repeated ColumnMeta columns = 4; + // List of primary key columns + repeated string primary_key = 5; + // Attributes + map attributes = 6; +} diff --git a/src/client/table/impl/table_client.cpp b/src/client/table/impl/table_client.cpp index ced55e4350..3e63550089 100644 --- a/src/client/table/impl/table_client.cpp +++ b/src/client/table/impl/table_client.cpp @@ -613,6 +613,36 @@ TAsyncDescribeExternalTableResult TTableClient::TImpl::DescribeExternalTable(con return promise.GetFuture(); } +TAsyncDescribeSystemViewResult TTableClient::TImpl::DescribeSystemView(const std::string& path, + const TDescribeSystemViewSettings& settings) +{ + auto request = MakeOperationRequest(settings); + request.set_path(path); + + auto promise = NewPromise(); + + auto extractor = [promise, settings](google::protobuf::Any* any, TPlainStatus status) mutable { + Ydb::Table::DescribeSystemViewResult proto; + if (any) { + any->UnpackTo(&proto); + } + promise.SetValue(TDescribeSystemViewResult(TStatus(std::move(status)), std::move(proto))); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::Table::V1::TableService::Stub::AsyncDescribeSystemView, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings) + ); + + return promise.GetFuture(); +} + TAsyncPrepareQueryResult TTableClient::TImpl::PrepareDataQuery(const TSession& session, const std::string& query, const TPrepareDataQuerySettings& settings) { diff --git a/src/client/table/impl/table_client.h b/src/client/table/impl/table_client.h index 034fb4eb57..3ae6ee7bb3 100644 --- a/src/client/table/impl/table_client.h +++ b/src/client/table/impl/table_client.h @@ -70,6 +70,7 @@ class TTableClient::TImpl: public TClientImplCommon, public TAsyncDescribeTableResult DescribeTable(const std::string& sessionId, const std::string& path, const TDescribeTableSettings& settings); TAsyncDescribeExternalDataSourceResult DescribeExternalDataSource(const std::string& path, const TDescribeExternalDataSourceSettings& settings); TAsyncDescribeExternalTableResult DescribeExternalTable(const std::string& path, const TDescribeExternalTableSettings& settings); + TAsyncDescribeSystemViewResult DescribeSystemView(const std::string& path, const TDescribeSystemViewSettings& settings); template TAsyncDataQueryResult ExecuteDataQuery(TSession& session, const std::string& query, const TTxControl& txControl, diff --git a/src/client/table/proto_accessor.cpp b/src/client/table/proto_accessor.cpp index 131c50943f..d4e69ccfd1 100644 --- a/src/client/table/proto_accessor.cpp +++ b/src/client/table/proto_accessor.cpp @@ -20,6 +20,10 @@ const Ydb::Table::DescribeExternalTableResult& TProtoAccessor::GetProto(const NT return description.GetProto(); } +const Ydb::Table::DescribeSystemViewResult& TProtoAccessor::GetProto(const NTable::TSystemViewDescription& description) { + return description.GetProto(); +} + NTable::TQueryStats TProtoAccessor::FromProto(const Ydb::TableStats::QueryStats& queryStats) { return NTable::TQueryStats(queryStats); } diff --git a/src/client/table/table.cpp b/src/client/table/table.cpp index 0072d9c88d..7be5bd85fc 100644 --- a/src/client/table/table.cpp +++ b/src/client/table/table.cpp @@ -1802,6 +1802,12 @@ TAsyncDescribeExternalTableResult TSession::DescribeExternalTable(const std::str return Client_->DescribeExternalTable(path, settings); } +TAsyncDescribeSystemViewResult TSession::DescribeSystemView(const std::string& path, + const TDescribeSystemViewSettings& settings) +{ + return Client_->DescribeSystemView(path, settings); +} + TAsyncDataQueryResult TSession::ExecuteDataQuery(const std::string& query, const TTxControl& txControl, const TExecDataQuerySettings& settings) { @@ -3417,5 +3423,123 @@ TExternalTableDescription TDescribeExternalTableResult::GetExternalTableDescript return ExternalTableDescription_; } +//////////////////////////////////////////////////////////////////////////////// + +class TSystemViewDescription::TImpl { + + TImpl(const Ydb::Table::DescribeSystemViewResult& proto) + { + // system view id + SysViewId_ = proto.sys_view_id(); + SysViewName_ = proto.sys_view_name(); + + // primary key + for (const auto& pk : proto.primary_key()) { + PrimaryKey_.push_back(pk); + } + + // columns + for (const auto& col : proto.columns()) { + std::optional not_null; + if (col.has_not_null()) { + not_null = col.not_null(); + } + + Columns_.emplace_back(col.name(), col.type(), "", not_null); + } + + // attributes + for (auto [key, value] : proto.attributes()) { + Attributes_[key] = value; + } + } + +public: + TImpl() = default; + + TImpl(Ydb::Table::DescribeSystemViewResult&& desc) + : TImpl(desc) + { + Proto_ = std::move(desc); + } + + const Ydb::Table::DescribeSystemViewResult& GetProto() const { + return Proto_; + } + + uint64_t GetSysViewId() const { + return SysViewId_; + } + + const std::string& GetSysViewName() const { + return SysViewName_; + } + + const std::vector& GetPrimaryKeyColumns() const { + return PrimaryKey_; + } + + const std::vector& GetColumns() const { + return Columns_; + } + + const std::unordered_map& GetAttributes() const { + return Attributes_; + } + +private: + Ydb::Table::DescribeSystemViewResult Proto_; + + uint64_t SysViewId_; + std::string SysViewName_; + std::vector PrimaryKey_; + std::vector Columns_; + std::unordered_map Attributes_; +}; + +TSystemViewDescription::TSystemViewDescription() + : Impl_(new TImpl) +{ +} + +TSystemViewDescription::TSystemViewDescription(Ydb::Table::DescribeSystemViewResult&& desc) + : Impl_(new TImpl(std::move(desc))) +{ +} + +uint64_t TSystemViewDescription::GetSysViewId() const { + return Impl_->GetSysViewId(); +} + +const std::string& TSystemViewDescription::GetSysViewName() const { + return Impl_->GetSysViewName(); +} + +const std::vector& TSystemViewDescription::GetPrimaryKeyColumns() const { + return Impl_->GetPrimaryKeyColumns(); +} + +std::vector TSystemViewDescription::GetTableColumns() const { + return Impl_->GetColumns(); +} + +const std::unordered_map& TSystemViewDescription::GetAttributes() const { + return Impl_->GetAttributes(); +} + +const Ydb::Table::DescribeSystemViewResult& TSystemViewDescription::GetProto() const { + return Impl_->GetProto(); +} + +TDescribeSystemViewResult::TDescribeSystemViewResult(TStatus&& status, Ydb::Table::DescribeSystemViewResult&& desc) + : NScheme::TDescribePathResult(std::move(status), desc.self()) + , SystemViewDescription_(std::move(desc)) +{} + +TSystemViewDescription TDescribeSystemViewResult::GetSystemViewDescription() const { + CheckStatusOk("TDescribeSystemViewResult::GetSystemViewDescription"); + return SystemViewDescription_; +} + } // namespace NTable } // namespace NYdb From e603485ef29c66c15f8f7161ac64776153ce3d9c Mon Sep 17 00:00:00 2001 From: vporyadke Date: Wed, 27 Aug 2025 14:26:51 +0000 Subject: [PATCH 03/25] check clock skew within pile and other bridge mode healthcheck updates (#21404) --- .github/last_commit.txt | 2 +- src/api/protos/ydb_monitoring.proto | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index a3cbe7e8a0..eb87d134b2 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -2b8366c9a860fe12b5e3aa62a1585e1d0a1a2686 +ea0ff3e7ef21a77d7cf161ddf5a2bfe983cbc826 diff --git a/src/api/protos/ydb_monitoring.proto b/src/api/protos/ydb_monitoring.proto index e012c61b90..704c9ce9ac 100644 --- a/src/api/protos/ydb_monitoring.proto +++ b/src/api/protos/ydb_monitoring.proto @@ -142,6 +142,7 @@ message LocationStorageVDisk { message LocationStorageGroup { repeated string id = 1; LocationStorageVDisk vdisk = 2; + LocationBridgePile pile = 3; } message LocationStoragePool { @@ -169,11 +170,16 @@ message LocationComputeSchema { string path = 2; } +message LocationBridgePile { + string name = 1; +} + message LocationCompute { LocationNode node = 1; LocationComputePool pool = 2; LocationComputeTablet tablet = 3; LocationComputeSchema schema = 4; + LocationBridgePile pile = 5; } message LocationDatabase { From 549f457092ac622e409cb8413de71d53589c9ef3 Mon Sep 17 00:00:00 2001 From: Andrey Molotkov Date: Wed, 27 Aug 2025 14:26:57 +0000 Subject: [PATCH 04/25] Support XDS protocols for client load balancing in order to request to Access Service (#21280) --- .github/last_commit.txt | 2 +- src/library/grpc/client/grpc_common.h | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index eb87d134b2..8bff621b56 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -ea0ff3e7ef21a77d7cf161ddf5a2bfe983cbc826 +0ca3d12fc88d0eadd10bafe124af11ac164ac6e9 diff --git a/src/library/grpc/client/grpc_common.h b/src/library/grpc/client/grpc_common.h index ea2f7ce68c..ee96c907ec 100644 --- a/src/library/grpc/client/grpc_common.h +++ b/src/library/grpc/client/grpc_common.h @@ -22,12 +22,13 @@ struct TGRpcClientConfig { ui32 MaxInFlight = 0; bool EnableSsl = false; grpc::SslCredentialsOptions SslCredentials; - grpc_compression_algorithm CompressionAlgoritm = GRPC_COMPRESS_NONE; + grpc_compression_algorithm CompressionAlgorithm = GRPC_COMPRESS_NONE; ui64 MemQuota = 0; std::unordered_map StringChannelParams; std::unordered_map IntChannelParams; std::string LoadBalancingPolicy = { }; std::string SslTargetNameOverride = { }; + bool UseXds = false; TGRpcClientConfig() = default; TGRpcClientConfig(const TGRpcClientConfig&) = default; @@ -46,7 +47,8 @@ struct TGRpcClientConfig { , SslCredentials{.pem_root_certs = NYdb::TStringType{caCert}, .pem_private_key = NYdb::TStringType{clientPrivateKey}, .pem_cert_chain = NYdb::TStringType{clientCert}} - , CompressionAlgoritm(compressionAlgorithm) + , CompressionAlgorithm(compressionAlgorithm) + , UseXds((Locator.starts_with("xds:///"))) {} }; @@ -54,7 +56,7 @@ inline std::shared_ptr CreateChannelInterface(const TGRp grpc::ChannelArguments args; args.SetMaxReceiveMessageSize(config.MaxInboundMessageSize ? config.MaxInboundMessageSize : config.MaxMessageSize); args.SetMaxSendMessageSize(config.MaxOutboundMessageSize ? config.MaxOutboundMessageSize : config.MaxMessageSize); - args.SetCompressionAlgorithm(config.CompressionAlgoritm); + args.SetCompressionAlgorithm(config.CompressionAlgorithm); for (const auto& kvp: config.StringChannelParams) { args.SetString(NYdb::TStringType{kvp.first}, NYdb::TStringType{kvp.second}); @@ -78,11 +80,16 @@ inline std::shared_ptr CreateChannelInterface(const TGRp if (!config.SslTargetNameOverride.empty()) { args.SetSslTargetNameOverride(NYdb::TStringType{config.SslTargetNameOverride}); } + std::shared_ptr channelCredentials = nullptr; if (config.EnableSsl || !config.SslCredentials.pem_root_certs.empty()) { - return grpc::CreateCustomChannel(grpc::string(config.Locator), grpc::SslCredentials(config.SslCredentials), args); + channelCredentials = grpc::SslCredentials(config.SslCredentials); } else { - return grpc::CreateCustomChannel(grpc::string(config.Locator), grpc::InsecureChannelCredentials(), args); + channelCredentials = grpc::InsecureChannelCredentials(); } + if (config.UseXds) { + channelCredentials = grpc::XdsCredentials(channelCredentials); + } + return grpc::CreateCustomChannel(grpc::string(config.Locator), channelCredentials, args); } } From 915fb159bbbdc37bbe42f08e612b746d0344d15d Mon Sep 17 00:00:00 2001 From: mregrock Date: Wed, 27 Aug 2025 14:27:09 +0000 Subject: [PATCH 05/25] Refactor bridge api according to negotiation (#22264) --- .github/last_commit.txt | 2 +- src/api/protos/draft/ydb_bridge.proto | 84 +++++++++++++++++++++------ 2 files changed, 67 insertions(+), 19 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 8bff621b56..a43b9ce984 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -0ca3d12fc88d0eadd10bafe124af11ac164ac6e9 +dd5bf98bacbbcc857fbfa7c5363e32a5b8825391 diff --git a/src/api/protos/draft/ydb_bridge.proto b/src/api/protos/draft/ydb_bridge.proto index 196b57e5b8..e6b0913137 100644 --- a/src/api/protos/draft/ydb_bridge.proto +++ b/src/api/protos/draft/ydb_bridge.proto @@ -7,21 +7,27 @@ option java_package = "com.yandex.ydb.bridge.proto"; option java_outer_classname = "BridgeProtos"; option java_multiple_files = true; -import "src/api/protos/ydb_operation.proto"; +import "src/api/protos/annotations/validation.proto"; +import "src/api/protos/ydb_operation.proto"; +import "google/protobuf/timestamp.proto"; -// State of a pile in relation to primary -enum PileState { - DISCONNECTED = 0; // disconnected from the cluster (no connectivity to other piles) - NOT_SYNCHRONIZED = 1; // not synchronized with primary, cannot be promoted - SYNCHRONIZED = 2; // fully synchronized with primary, may be promoted - PROMOTE = 3; // pile is being promoted to primary - PRIMARY = 4; // pile is primary -} - -// Pair of pile id and pile state -message PileStateUpdate { - uint32 pile_id = 1; - PileState state = 2; +// Describes the state of a single pile within the cluster +message PileState { + // State of a pile + enum State { + UNSPECIFIED = 0; + PRIMARY = 1; // pile is primary + PROMOTE = 2; // pile is being promoted to primary + SYNCHRONIZED = 3; // fully synchronized with primary, may be promoted + NOT_SYNCHRONIZED = 4; // not synchronized with primary, cannot be promoted + SUSPENDED = 5; // pile is gracefully taken down, preparing to be disconnected + DISCONNECTED = 6; // disconnected from the cluster (no connectivity to other piles) + } + + // The name of the pile + string pile_name = 1 [(length).le = 128]; + // The operational state of the pile + State state = 2; } message GetClusterStateRequest { @@ -34,15 +40,17 @@ message GetClusterStateResponse { message GetClusterStateResult { // Current cluster state - repeated PileStateUpdate per_pile_state = 1; + repeated PileState pile_states = 1; } message UpdateClusterStateRequest { Ydb.Operations.OperationParams operation_params = 1; // List of desired pile states to update - repeated PileStateUpdate updates = 2; - // If set, acquire quorum only for specific pile(s) - repeated uint32 specific_pile_ids = 3; + repeated PileState updates = 2 [(size).ge = 1]; + // List of piles that are required to form a quorum for this update to succeed. + // These piles act as "witnesses" for the operation. + // If empty, all piles in the cluster will be used for the quorum check. + repeated string quorum_piles = 3; } message UpdateClusterStateResponse { @@ -51,3 +59,43 @@ message UpdateClusterStateResponse { message UpdateClusterStateResult { } + +message BridgeHistoryEvent { + // Timestamp of the event + google.protobuf.Timestamp timestamp = 1; + // UUID of the request that initiated the change + string request_id = 2 [(length).le = 64]; + // UUID of the cluster state generation + string generation_uuid = 3 [(length).le = 64]; + // The primary pile at the time of the event + string primary_pile = 4 [(length).le = 128]; + // Name of the pile that changed state + string pile_name = 5 [(length).le = 128]; + // Event type + string event = 6; + // State before the change + PileState.State from_state = 7; + // State after the change + PileState.State to_state = 8; + // Who initiated the change (e.g., "admin" or "system") + string initiator = 9; +} + +message GetClusterHistoryRequest { + Ydb.Operations.OperationParams operation_params = 1; + // Limit the number of history records to show + uint64 limit = 2; + // Pagination token + string page_token = 3 [(length).le = 256]; +} + +message GetClusterHistoryResponse { + Ydb.Operations.Operation operation = 1; +} + +message GetClusterHistoryResult { + // List of history events + repeated BridgeHistoryEvent events = 1; + // Pagination token + string next_page_token = 2; +} From 6448c6b2cf6f9bccc68a35b7a325bf283d4971f3 Mon Sep 17 00:00:00 2001 From: Ilia Shakhov Date: Wed, 27 Aug 2025 14:27:15 +0000 Subject: [PATCH 06/25] Add incremental backup to operation API (#21848) --- .github/last_commit.txt | 2 +- .../library/operation_id/operation_id.h | 1 + src/api/protos/draft/ydb_backup.proto | 19 +++++++++++++++++++ src/client/operation/operation.cpp | 12 ++++++++++++ src/library/operation_id/operation_id.cpp | 7 +++++++ .../operation_id/protos/operation_id.proto | 1 + 6 files changed, 41 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index a43b9ce984..6a68d1afc5 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -dd5bf98bacbbcc857fbfa7c5363e32a5b8825391 +1f0d766a206a5e8f2dd5b05a01bd1e9ded3ca52e diff --git a/include/ydb-cpp-sdk/library/operation_id/operation_id.h b/include/ydb-cpp-sdk/library/operation_id/operation_id.h index a8d5411cb5..cdcce79e39 100644 --- a/include/ydb-cpp-sdk/library/operation_id/operation_id.h +++ b/include/ydb-cpp-sdk/library/operation_id/operation_id.h @@ -26,6 +26,7 @@ class TOperationId { IMPORT = 8, SCRIPT_EXECUTION = 9, SS_BG_TASKS = 10, + INCREMENTAL_BACKUP = 11, }; struct TData { diff --git a/src/api/protos/draft/ydb_backup.proto b/src/api/protos/draft/ydb_backup.proto index 00b6489395..ec7d7c8864 100644 --- a/src/api/protos/draft/ydb_backup.proto +++ b/src/api/protos/draft/ydb_backup.proto @@ -177,3 +177,22 @@ message DeleteBackupCollectionResponse { message DeleteBackupCollectionResult { BackupCollection backup_collection = 1; } + +message BackupProgress { + enum Progress { + PROGRESS_UNSPECIFIED = 0; + PROGRESS_PREPARING = 1; + PROGRESS_TRANSFER_DATA = 2; + PROGRESS_DONE = 3; + PROGRESS_CANCELLATION = 4; + PROGRESS_CANCELLED = 5; + } +} + +message IncrementalBackupMetadata { + BackupProgress.Progress progress = 1; + int32 progress_percent = 2 [(Ydb.value) = "[0; 100]"]; +} + +message IncrementalBackupResult { +} diff --git a/src/client/operation/operation.cpp b/src/client/operation/operation.cpp index 059fa9d2d8..a9ed76d202 100644 --- a/src/client/operation/operation.cpp +++ b/src/client/operation/operation.cpp @@ -11,6 +11,10 @@ #include #include +#ifdef YDB_SDK_USE_DRAFT_API +#include +#endif + #include #include #include @@ -206,5 +210,13 @@ TFuture> TOperationClient::Li return List("scriptexec", pageSize, pageToken); } +#ifdef YDB_SDK_USE_DRAFT_API +template TFuture TOperationClient::Get(const TOperation::TOperationId& id); +template <> +TFuture> TOperationClient::List(ui64 pageSize, const std::string& pageToken) { + return List("incbackup", pageSize, pageToken); +} +#endif + } // namespace NOperation } // namespace NYdb diff --git a/src/library/operation_id/operation_id.cpp b/src/library/operation_id/operation_id.cpp index 3a62e66ed9..d9d667607d 100644 --- a/src/library/operation_id/operation_id.cpp +++ b/src/library/operation_id/operation_id.cpp @@ -66,6 +66,9 @@ std::string ProtoToString(const Ydb::TOperationId& proto) { case Ydb::TOperationId::SCRIPT_EXECUTION: res << "ydb://scriptexec"; break; + case Ydb::TOperationId::INCREMENTAL_BACKUP: + res << "ydb://incbackup"; + break; default: Y_ABORT_UNLESS(false, "unexpected kind"); } @@ -305,6 +308,10 @@ TOperationId::EKind ParseKind(const std::string_view value) { return TOperationId::SCRIPT_EXECUTION; } + if (value.starts_with("incbackup")) { + return TOperationId::INCREMENTAL_BACKUP; + } + return TOperationId::UNUSED; } diff --git a/src/library/operation_id/protos/operation_id.proto b/src/library/operation_id/protos/operation_id.proto index 0be1c0ad36..5e8a2f6ff0 100644 --- a/src/library/operation_id/protos/operation_id.proto +++ b/src/library/operation_id/protos/operation_id.proto @@ -15,6 +15,7 @@ message TOperationId { IMPORT = 8; SCRIPT_EXECUTION = 9; SS_BG_TASKS = 10; + INCREMENTAL_BACKUP = 11; } message TData { From 7863848fe57dcc5d673cefea13ec324cb50a5110 Mon Sep 17 00:00:00 2001 From: qyryq Date: Wed, 27 Aug 2025 14:27:22 +0000 Subject: [PATCH 07/25] Topic SDK: fix in direct reader (#22539) --- .github/last_commit.txt | 2 +- src/client/topic/impl/direct_reader.cpp | 13 +++++++------ tests/integration/topic/direct_read.cpp | 7 +++++++ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 6a68d1afc5..4097ff924f 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -1f0d766a206a5e8f2dd5b05a01bd1e9ded3ca52e +604a3ecfc4f424dfe1d362b641c2ed50174f5350 diff --git a/src/client/topic/impl/direct_reader.cpp b/src/client/topic/impl/direct_reader.cpp index 7e84cd0ea5..c58d7de977 100644 --- a/src/client/topic/impl/direct_reader.cpp +++ b/src/client/topic/impl/direct_reader.cpp @@ -731,12 +731,6 @@ void TDirectReadSession::OnReadDoneImpl(Ydb::Topic::StreamDirectReadMessage::Sta auto partitionSessionId = response.partition_session_id(); auto it = PartitionSessions.find(partitionSessionId); - if (it->second.Location.GetGeneration() != response.generation()) { - LOG_LAZY(Log, TLOG_DEBUG, GetLogPrefix() << "Got StartDirectReadPartitionSessionResponse for wrong generation " - << "(expected " << it->second.Location.GetGeneration() - << ", got " << response.generation() << ") partition_session_id=" << partitionSessionId); - return; - } if (it == PartitionSessions.end()) { // We could get a StopPartitionSessionRequest from server before processing this response. @@ -744,6 +738,13 @@ void TDirectReadSession::OnReadDoneImpl(Ydb::Topic::StreamDirectReadMessage::Sta return; } + if (it->second.Location.GetGeneration() != response.generation()) { + LOG_LAZY(Log, TLOG_DEBUG, GetLogPrefix() << "Got StartDirectReadPartitionSessionResponse for wrong generation " + << "(expected " << it->second.Location.GetGeneration() + << ", got " << response.generation() << ") partition_session_id=" << partitionSessionId); + return; + } + auto& partitionSession = it->second; auto transitioned = partitionSession.TransitionTo(TDirectReadPartitionSession::EState::WORKING); diff --git a/tests/integration/topic/direct_read.cpp b/tests/integration/topic/direct_read.cpp index 15673d44db..2e3d1debb7 100644 --- a/tests/integration/topic/direct_read.cpp +++ b/tests/integration/topic/direct_read.cpp @@ -1292,6 +1292,13 @@ TEST_F(DirectReadWithControlSession, StopPartitionSession) { } } + { + // Send a StartDirectReadPartitionSessionResponse with a random partition session id, + // the session should ignore it. + auto r = TMockDirectReadSessionProcessor::TServerReadInfo(); + setup.AddDirectReadResponse(r.StartDirectReadPartitionSessionResponse(333)); + } + setup.GetControlSession()->Start(); { auto r = TMockReadSessionProcessor::TServerReadInfo(); From 909a6a169c92e5ac3c29fc6fc628ff211dc17761 Mon Sep 17 00:00:00 2001 From: Bulat Date: Wed, 27 Aug 2025 14:27:28 +0000 Subject: [PATCH 08/25] [C++ SDK] Update vector example (#22613) --- .github/last_commit.txt | 2 +- examples/vector_index_builtin/main.cpp | 6 +- .../vector_index_builtin/vector_index.cpp | 107 +++++++++++++++++- examples/vector_index_builtin/vector_index.h | 17 ++- 4 files changed, 124 insertions(+), 8 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 4097ff924f..82cc737e1a 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -604a3ecfc4f424dfe1d362b641c2ed50174f5350 +656353ad7d790ab345e181510c8fdec9e962a3a5 diff --git a/examples/vector_index_builtin/main.cpp b/examples/vector_index_builtin/main.cpp index 198c7f4944..023c9f149d 100644 --- a/examples/vector_index_builtin/main.cpp +++ b/examples/vector_index_builtin/main.cpp @@ -39,10 +39,10 @@ void VectorExample( {.Id = "8", .Document = "document 8", .Embedding = {0.02, 0.98, 0.1}}, {.Id = "9", .Document = "document 9", .Embedding = {0.0, 1.0, 0.05}}, }; - InsertItems(client, tableName, items); - PrintResults(SearchItems(client, tableName, {1.0, 0.0, 0.0}, "CosineSimilarity", 3)); + InsertItemsAsBytes(client, tableName, items); + PrintResults(SearchItemsAsBytes(client, tableName, {1.0, 0.0, 0.0}, "CosineSimilarity", 3)); AddIndex(driver, client, database, tableName, indexName, "similarity=cosine", 3, 1, 3); - PrintResults(SearchItems(client, tableName, {1.0, 0.0, 0.0}, "CosineSimilarity", 3, indexName)); + PrintResults(SearchItemsAsBytes(client, tableName, {1.0, 0.0, 0.0}, "CosineSimilarity", 3, indexName)); } catch (const std::exception& e) { std::cerr << "Execution failed: " << e.what() << std::endl; } diff --git a/examples/vector_index_builtin/vector_index.cpp b/examples/vector_index_builtin/vector_index.cpp index 2df8d3e19d..67cf2557f6 100644 --- a/examples/vector_index_builtin/vector_index.cpp +++ b/examples/vector_index_builtin/vector_index.cpp @@ -2,6 +2,15 @@ #include +std::string ConvertVectorToBytes(const std::vector& vector) +{ + std::string result; + for (const auto& value : vector) { + const char* bytes = reinterpret_cast(&value); + result += std::string(bytes, sizeof(float)); + } + return result + "\x01"; +} void DropVectorTable(NYdb::NQuery::TQueryClient& client, const std::string& tableName) { @@ -29,7 +38,52 @@ void CreateVectorTable(NYdb::NQuery::TQueryClient& client, const std::string& ta std::cout << "Vector table created: " << tableName << std::endl; } -void InsertItems( +void InsertItemsAsBytes( + NYdb::NQuery::TQueryClient& client, + const std::string& tableName, + const std::vector& items) +{ + std::string query = std::format(R"( + DECLARE $items AS List>; + UPSERT INTO `{0}` + ( + id, + document, + embedding + ) + SELECT + id, + document, + embedding, + FROM AS_TABLE($items); + )", tableName); + + NYdb::TParamsBuilder paramsBuilder; + auto& valueBuilder = paramsBuilder.AddParam("$items"); + valueBuilder.BeginList(); + for (const auto& item : items) { + valueBuilder.AddListItem(); + valueBuilder.BeginStruct(); + valueBuilder.AddMember("id").Utf8(item.Id); + valueBuilder.AddMember("document").Utf8(item.Document); + valueBuilder.AddMember("embedding").String(ConvertVectorToBytes(item.Embedding)); + valueBuilder.EndStruct(); + } + valueBuilder.EndList(); + valueBuilder.Build(); + + NYdb::NStatusHelpers::ThrowOnError(client.RetryQuerySync([params = paramsBuilder.Build(), &query](NYdb::NQuery::TSession session) { + return session.ExecuteQuery(query, NYdb::NQuery::TTxControl::BeginTx(NYdb::NQuery::TTxSettings::SerializableRW()).CommitTx(), params).ExtractValueSync(); + })); + + std::cout << items.size() << " items inserted" << std::endl; +} + +void InsertItemsAsFloatList( NYdb::NQuery::TQueryClient& client, const std::string& tableName, const std::vector& items) @@ -122,7 +176,56 @@ void AddIndex( std::cout << "Table index `" << indexName << "` for table `" << tableName << "` added" << std::endl; } -std::vector SearchItems( +std::vector SearchItemsAsBytes( + NYdb::NQuery::TQueryClient& client, + const std::string& tableName, + const std::vector& embedding, + const std::string& strategy, + std::uint64_t limit, + const std::optional& indexName) +{ + std::string viewIndex = indexName ? "VIEW " + *indexName : ""; + std::string sortOrder = strategy.ends_with("Similarity") ? "DESC" : "ASC"; + + std::string query = std::format(R"( + DECLARE $embedding as String; + SELECT + id, + document, + Knn::{2}(embedding, $embedding) as score + FROM {0} {1} + ORDER BY score + {3} + LIMIT {4}; + )", tableName, viewIndex, strategy, sortOrder, limit); + + auto params = NYdb::TParamsBuilder() + .AddParam("$embedding") + .String(ConvertVectorToBytes(embedding)) + .Build() + .Build(); + + std::vector result; + + NYdb::NStatusHelpers::ThrowOnError(client.RetryQuerySync([params, &query, &result](NYdb::NQuery::TSession session) { + auto execResult = session.ExecuteQuery(query, NYdb::NQuery::TTxControl::BeginTx(NYdb::NQuery::TTxSettings::SerializableRW()).CommitTx(), params).ExtractValueSync(); + if (execResult.IsSuccess()) { + auto parser = execResult.GetResultSetParser(0); + while (parser.TryNextRow()) { + result.push_back({ + .Id = *parser.ColumnParser(0).GetOptionalUtf8(), + .Document = *parser.ColumnParser(1).GetOptionalUtf8(), + .Score = *parser.ColumnParser(2).GetOptionalFloat() + }); + } + } + return execResult; + })); + + return result; +} + +std::vector SearchItemsAsFloatList( NYdb::NQuery::TQueryClient& client, const std::string& tableName, const std::vector& embedding, diff --git a/examples/vector_index_builtin/vector_index.h b/examples/vector_index_builtin/vector_index.h index 762f42b941..f728c2dd87 100644 --- a/examples/vector_index_builtin/vector_index.h +++ b/examples/vector_index_builtin/vector_index.h @@ -22,7 +22,12 @@ void DropVectorTable(NYdb::NQuery::TQueryClient& client, const std::string& tabl void CreateVectorTable(NYdb::NQuery::TQueryClient& client, const std::string& tableName); -void InsertItems( +void InsertItemsAsBytes( + NYdb::NQuery::TQueryClient& client, + const std::string& tableName, + const std::vector& items); + +void InsertItemsAsFloatList( NYdb::NQuery::TQueryClient& client, const std::string& tableName, const std::vector& items); @@ -38,7 +43,15 @@ void AddIndex( std::uint64_t levels, std::uint64_t clusters); -std::vector SearchItems( +std::vector SearchItemsAsBytes( + NYdb::NQuery::TQueryClient& client, + const std::string& tableName, + const std::vector& embedding, + const std::string& strategy, + std::uint64_t limit, + const std::optional& indexName = std::nullopt); + +std::vector SearchItemsAsFloatList( NYdb::NQuery::TQueryClient& client, const std::string& tableName, const std::vector& embedding, From 7d92f6ddec7011b280978a52c6fbff0b7f3f424b Mon Sep 17 00:00:00 2001 From: Nikolay Shestakov Date: Wed, 27 Aug 2025 14:27:34 +0000 Subject: [PATCH 09/25] Fixed concurrent modification of the HierarchyData (#22663) --- .github/last_commit.txt | 2 +- src/client/topic/impl/read_session_impl.h | 3 ++- src/client/topic/impl/read_session_impl.ipp | 6 ++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 82cc737e1a..22bc9a3700 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -656353ad7d790ab345e181510c8fdec9e962a3a5 +f0e65f2981c17d0e0330f7e27b91a422741e5339 diff --git a/src/client/topic/impl/read_session_impl.h b/src/client/topic/impl/read_session_impl.h index 5855780b6b..60d313496b 100644 --- a/src/client/topic/impl/read_session_impl.h +++ b/src/client/topic/impl/read_session_impl.h @@ -1427,7 +1427,7 @@ class TSingleClusterReadSessionImpl : public TEnableSelfContext::IRetryState::TPtr RetryState; // Current retry state (if now we are (re)connecting). size_t ConnectionAttemptsDone = 0; @@ -1459,6 +1459,7 @@ class TSingleClusterReadSessionImpl : public TEnableSelfContext> HierarchyData; std::unordered_set ReadingFinishedData; diff --git a/src/client/topic/impl/read_session_impl.ipp b/src/client/topic/impl/read_session_impl.ipp index 72293e5da9..382a7b9cf6 100644 --- a/src/client/topic/impl/read_session_impl.ipp +++ b/src/client/topic/impl/read_session_impl.ipp @@ -2073,12 +2073,16 @@ bool TSingleClusterReadSessionImpl::TPartitionCookieMappin template void TSingleClusterReadSessionImpl::RegisterParentPartition(ui32 partitionId, ui32 parentPartitionId, ui64 parentPartitionSessionId) { + std::lock_guard guard(HierarchyDataLock); + auto& values = HierarchyData[partitionId]; values.push_back({parentPartitionId, parentPartitionSessionId}); } template void TSingleClusterReadSessionImpl::UnregisterPartition(ui32 partitionId, ui64 partitionSessionId) { + std::lock_guard guard(HierarchyDataLock); + for (auto it = HierarchyData.begin(); it != HierarchyData.end();) { auto& values = it->second; for (auto v = values.begin(); v != values.end();) { @@ -2098,6 +2102,8 @@ void TSingleClusterReadSessionImpl::UnregisterPartition(ui template std::vector TSingleClusterReadSessionImpl::GetParentPartitionSessions(ui32 partitionId, ui64 partitionSessionId) { + std::lock_guard guard(HierarchyDataLock); + auto it = HierarchyData.find(partitionId); if (it == HierarchyData.end()) { return {}; From 8a7ce77df99037782c448db973890750054dbee7 Mon Sep 17 00:00:00 2001 From: Bulat Date: Wed, 27 Aug 2025 14:27:40 +0000 Subject: [PATCH 10/25] Added pile info in Discovery/ListEndpoints response (#22549) --- .github/last_commit.txt | 2 +- .../ydb-cpp-sdk/client/discovery/discovery.h | 19 ++++++++++++ src/api/protos/draft/ydb_bridge.proto | 20 +------------ src/api/protos/ydb_bridge_common.proto | 29 +++++++++++++++++++ src/api/protos/ydb_discovery.proto | 10 +++++++ src/client/discovery/discovery.cpp | 13 +++++++++ 6 files changed, 73 insertions(+), 20 deletions(-) create mode 100644 src/api/protos/ydb_bridge_common.proto diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 22bc9a3700..0f80e77f2d 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -f0e65f2981c17d0e0330f7e27b91a422741e5339 +2c39c0acafed011ae79a1845b572749e80323f39 diff --git a/include/ydb-cpp-sdk/client/discovery/discovery.h b/include/ydb-cpp-sdk/client/discovery/discovery.h index 5313113376..41c7c8736c 100644 --- a/include/ydb-cpp-sdk/client/discovery/discovery.h +++ b/include/ydb-cpp-sdk/client/discovery/discovery.h @@ -62,14 +62,33 @@ struct TEndpointInfo { std::vector IPv4Addrs; std::vector IPv6Addrs; std::string SslTargetNameOverride; + std::string BridgePileName; +}; + +struct TPileState { + enum EState { + UNSPECIFIED = 0 /* "unspecified" */, + PRIMARY = 1 /* "primary" */, + PROMOTE = 2 /* "promote" */, + SYNCHRONIZED = 3 /* "synchronized" */, + NOT_SYNCHRONIZED = 4 /* "not_synchronized" */, + SUSPENDED = 5 /* "suspended" */, + DISCONNECTED = 6 /* "disconnected" */ + }; + + EState State; + std::string PileName; }; class TListEndpointsResult : public TStatus { public: TListEndpointsResult(TStatus&& status, const Ydb::Discovery::ListEndpointsResult& endpoints); const std::vector& GetEndpointsInfo() const; + const std::vector& GetPileStates() const; + private: std::vector Info_; + std::vector PileStates_; }; using TAsyncListEndpointsResult = NThreading::TFuture; diff --git a/src/api/protos/draft/ydb_bridge.proto b/src/api/protos/draft/ydb_bridge.proto index e6b0913137..50c5daf692 100644 --- a/src/api/protos/draft/ydb_bridge.proto +++ b/src/api/protos/draft/ydb_bridge.proto @@ -8,28 +8,10 @@ option java_outer_classname = "BridgeProtos"; option java_multiple_files = true; import "src/api/protos/annotations/validation.proto"; +import "src/api/protos/ydb_bridge_common.proto"; import "src/api/protos/ydb_operation.proto"; import "google/protobuf/timestamp.proto"; -// Describes the state of a single pile within the cluster -message PileState { - // State of a pile - enum State { - UNSPECIFIED = 0; - PRIMARY = 1; // pile is primary - PROMOTE = 2; // pile is being promoted to primary - SYNCHRONIZED = 3; // fully synchronized with primary, may be promoted - NOT_SYNCHRONIZED = 4; // not synchronized with primary, cannot be promoted - SUSPENDED = 5; // pile is gracefully taken down, preparing to be disconnected - DISCONNECTED = 6; // disconnected from the cluster (no connectivity to other piles) - } - - // The name of the pile - string pile_name = 1 [(length).le = 128]; - // The operational state of the pile - State state = 2; -} - message GetClusterStateRequest { Ydb.Operations.OperationParams operation_params = 1; } diff --git a/src/api/protos/ydb_bridge_common.proto b/src/api/protos/ydb_bridge_common.proto new file mode 100644 index 0000000000..2e93dc7330 --- /dev/null +++ b/src/api/protos/ydb_bridge_common.proto @@ -0,0 +1,29 @@ +syntax = "proto3"; +option cc_enable_arenas = true; + +package Ydb.Bridge; + +option java_package = "com.yandex.ydb.bridge.proto"; +option java_outer_classname = "BridgeProtos"; +option java_multiple_files = true; + +import "src/api/protos/annotations/validation.proto"; + +// Describes the state of a single pile within the cluster +message PileState { + // State of a pile + enum State { + UNSPECIFIED = 0; + PRIMARY = 1; // pile is primary + PROMOTE = 2; // pile is being promoted to primary + SYNCHRONIZED = 3; // fully synchronized with primary, may be promoted + NOT_SYNCHRONIZED = 4; // not synchronized with primary, cannot be promoted + SUSPENDED = 5; // pile is gracefully taken down, preparing to be disconnected + DISCONNECTED = 6; // disconnected from the cluster (no connectivity to other piles) + } + + // The name of the pile + string pile_name = 1 [(length).le = 128]; + // The operational state of the pile + State state = 2; +} diff --git a/src/api/protos/ydb_discovery.proto b/src/api/protos/ydb_discovery.proto index 3cffe33774..003e43cec2 100644 --- a/src/api/protos/ydb_discovery.proto +++ b/src/api/protos/ydb_discovery.proto @@ -5,6 +5,7 @@ package Ydb.Discovery; option java_package = "com.yandex.ydb.discovery"; option java_outer_classname = "DiscoveryProtos"; +import "src/api/protos/ydb_bridge_common.proto"; import "src/api/protos/ydb_operation.proto"; message ListEndpointsRequest { @@ -34,11 +35,20 @@ message EndpointInfo { // endpoint is using certificate chain valid for a balancer hostname, and // not this specific node hostname. string ssl_target_name_override = 10; + + // EXPERIMENTAL + // Optional name of the bridge pile this endpoint belongs to. + optional string bridge_pile_name = 11; } message ListEndpointsResult { repeated EndpointInfo endpoints = 1; string self_location = 2; + + // EXPERIMENTAL + // List of bridge pile states. + // This field is empty if cluster is not in bridge mode. + repeated Ydb.Bridge.PileState pile_states = 3; } message ListEndpointsResponse { diff --git a/src/client/discovery/discovery.cpp b/src/client/discovery/discovery.cpp index f8326e6367..2da501ead7 100644 --- a/src/client/discovery/discovery.cpp +++ b/src/client/discovery/discovery.cpp @@ -31,6 +31,15 @@ TListEndpointsResult::TListEndpointsResult(TStatus&& status, const Ydb::Discover info.IPv6Addrs.emplace_back(addr); } info.SslTargetNameOverride = endpointInfo.ssl_target_name_override(); + info.BridgePileName = endpointInfo.bridge_pile_name(); + } + + const auto& pileStates = proto.pile_states(); + PileStates_.reserve(pileStates.size()); + for (const auto& pileState : pileStates) { + TPileState& info = PileStates_.emplace_back(); + info.State = static_cast(pileState.state()); + info.PileName = pileState.pile_name(); } } @@ -38,6 +47,10 @@ const std::vector& TListEndpointsResult::GetEndpointsInfo() const return Info_; } +const std::vector& TListEndpointsResult::GetPileStates() const { + return PileStates_; +} + TWhoAmIResult::TWhoAmIResult(TStatus&& status, const Ydb::Discovery::WhoAmIResult& proto) : TStatus(std::move(status)) { From 6b1f46d074b6cda0769d9d717c3921a5b679dfdf Mon Sep 17 00:00:00 2001 From: Bulat Date: Wed, 27 Aug 2025 14:27:46 +0000 Subject: [PATCH 11/25] [C++ SDK] Fixed vector example performance (#22838) --- .github/last_commit.txt | 2 +- examples/vector_index_builtin/vector_index.cpp | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 0f80e77f2d..404edd1f82 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -2c39c0acafed011ae79a1845b572749e80323f39 +295cd50b2a76b316c8b59f05fbb4fcb54b1f6a70 diff --git a/examples/vector_index_builtin/vector_index.cpp b/examples/vector_index_builtin/vector_index.cpp index 67cf2557f6..6796ed4884 100644 --- a/examples/vector_index_builtin/vector_index.cpp +++ b/examples/vector_index_builtin/vector_index.cpp @@ -4,12 +4,7 @@ std::string ConvertVectorToBytes(const std::vector& vector) { - std::string result; - for (const auto& value : vector) { - const char* bytes = reinterpret_cast(&value); - result += std::string(bytes, sizeof(float)); - } - return result + "\x01"; + return std::string{reinterpret_cast(vector.data()), vector.size() * sizeof(float)} + "\x01"; } void DropVectorTable(NYdb::NQuery::TQueryClient& client, const std::string& tableName) From 9bf8ff0719d481222036e308f1ab06e87cb71296 Mon Sep 17 00:00:00 2001 From: Bulat Date: Wed, 27 Aug 2025 14:27:52 +0000 Subject: [PATCH 12/25] Fixed java build for public API protos (#22901) --- .github/last_commit.txt | 2 +- src/api/protos/ydb_bridge_common.proto | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 404edd1f82..aa08352eab 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -295cd50b2a76b316c8b59f05fbb4fcb54b1f6a70 +33e3ad62ec6c0b966592cf92c0ad2be07db6d05a diff --git a/src/api/protos/ydb_bridge_common.proto b/src/api/protos/ydb_bridge_common.proto index 2e93dc7330..30cd2852fd 100644 --- a/src/api/protos/ydb_bridge_common.proto +++ b/src/api/protos/ydb_bridge_common.proto @@ -4,8 +4,6 @@ option cc_enable_arenas = true; package Ydb.Bridge; option java_package = "com.yandex.ydb.bridge.proto"; -option java_outer_classname = "BridgeProtos"; -option java_multiple_files = true; import "src/api/protos/annotations/validation.proto"; From 2adc4317ac0348a796dc44a186473e266f516a11 Mon Sep 17 00:00:00 2001 From: Daniil Timizhev Date: Wed, 27 Aug 2025 14:27:58 +0000 Subject: [PATCH 13/25] Configuration for the result set format in QueryService with Arrow (#22232) --- .github/last_commit.txt | 2 +- include/ydb-cpp-sdk/client/arrow/accessor.h | 16 ++++ include/ydb-cpp-sdk/client/query/query.h | 29 ++++++++ include/ydb-cpp-sdk/client/result/result.h | 30 ++++++++ src/api/protos/ydb_formats.proto | 42 +++++++++++ src/api/protos/ydb_query.proto | 23 ++++++ src/api/protos/ydb_value.proto | 26 ++++++- src/client/arrow/accessor.cpp | 17 +++++ src/client/query/impl/exec_query.cpp | 81 +++++++++++++++++++-- src/client/result/result.cpp | 60 ++++++++++++++- tests/integration/basic_example/main.cpp | 2 + 11 files changed, 313 insertions(+), 15 deletions(-) create mode 100644 include/ydb-cpp-sdk/client/arrow/accessor.h create mode 100644 src/client/arrow/accessor.cpp diff --git a/.github/last_commit.txt b/.github/last_commit.txt index aa08352eab..425801c2cc 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -33e3ad62ec6c0b966592cf92c0ad2be07db6d05a +d6599f50029614cb41dde51939369797b38400ba diff --git a/include/ydb-cpp-sdk/client/arrow/accessor.h b/include/ydb-cpp-sdk/client/arrow/accessor.h new file mode 100644 index 0000000000..4ea22648fe --- /dev/null +++ b/include/ydb-cpp-sdk/client/arrow/accessor.h @@ -0,0 +1,16 @@ +#pragma once + +#include + +namespace NYdb::inline V3 { + +//! Provides access to Arrow batches of result set. It is not recommended to use this +//! class in client applications as it is an experimental feature. +class TArrowAccessor { +public: + static TResultSet::EFormat Format(const TResultSet& resultSet); + static const std::string& GetArrowSchema(const TResultSet& resultSet); + static const std::vector& GetArrowBatches(const TResultSet& resultSet); +}; + +} // namespace NYdb diff --git a/include/ydb-cpp-sdk/client/query/query.h b/include/ydb-cpp-sdk/client/query/query.h index d6e8478ae6..ab12786495 100644 --- a/include/ydb-cpp-sdk/client/query/query.h +++ b/include/ydb-cpp-sdk/client/query/query.h @@ -36,6 +36,12 @@ enum class EStatsMode { Profile = 40, }; +enum class ESchemaInclusionMode { + Unspecified = 0, + Always = 1, + FirstOnly = 2, +}; + std::optional ParseStatsMode(std::string_view statsMode); std::string_view StatsModeToString(const EStatsMode statsMode); @@ -49,6 +55,26 @@ enum class EExecStatus { Failed = 50, }; +struct TArrowFormatSettings { + using TSelf = TArrowFormatSettings; + + struct TCompressionCodec { + using TSelf = TCompressionCodec; + + enum class EType { + Unspecified = 0, + None = 1, + Zstd = 2, + Lz4Frame = 3, + }; + + FLUENT_SETTING_DEFAULT(EType, Type, EType::Unspecified); + FLUENT_SETTING_OPTIONAL(int32_t, Level); + }; + + FLUENT_SETTING_OPTIONAL(TCompressionCodec, CompressionCodec); +}; + using TAsyncExecuteQueryPart = NThreading::TFuture; class TExecuteQueryIterator : public TStatus { @@ -84,6 +110,9 @@ struct TExecuteQuerySettings : public TRequestSettings { FLUENT_SETTING_OPTIONAL(bool, ConcurrentResultSets); FLUENT_SETTING(std::string, ResourcePool); FLUENT_SETTING_OPTIONAL(std::chrono::milliseconds, StatsCollectPeriod); + FLUENT_SETTING_DEFAULT(ESchemaInclusionMode, SchemaInclusionMode, ESchemaInclusionMode::Unspecified); + FLUENT_SETTING_DEFAULT(TResultSet::EFormat, Format, TResultSet::EFormat::Unspecified); + FLUENT_SETTING_OPTIONAL(TArrowFormatSettings, ArrowFormatSettings); }; struct TBeginTxSettings : public TRequestSettings {}; diff --git a/include/ydb-cpp-sdk/client/result/result.h b/include/ydb-cpp-sdk/client/result/result.h index 38772ef64a..79e64819d0 100644 --- a/include/ydb-cpp-sdk/client/result/result.h +++ b/include/ydb-cpp-sdk/client/result/result.h @@ -12,7 +12,12 @@ namespace Ydb { namespace NYdb::inline V3 { +namespace NQuery { + struct TExecuteQueryBuffer; +} + class TProtoAccessor; +class TArrowAccessor; struct TColumn { std::string Name; @@ -33,10 +38,23 @@ bool operator!=(const TColumn& col1, const TColumn& col2); class TResultSet { friend class TResultSetParser; friend class NYdb::TProtoAccessor; + friend class NYdb::TArrowAccessor; + friend struct NQuery::TExecuteQueryBuffer; + +public: + enum class EFormat { + Unspecified = 0, + Value = 1, + Arrow = 2, + }; + public: TResultSet(const Ydb::ResultSet& proto); TResultSet(Ydb::ResultSet&& proto); + TResultSet(const Ydb::ResultSet& proto, const std::string& arrowSchema, const std::vector& bytesData); + TResultSet(Ydb::ResultSet&& proto, std::string&& arrowSchema, std::vector&& bytesData); + //! Returns number of columns size_t ColumnsCount() const; @@ -52,6 +70,18 @@ class TResultSet { private: const Ydb::ResultSet& GetProto() const; + //! Mutable proto is used to move bytes without copying + Ydb::ResultSet& MutableProto(); + + //! Returns format of the result set + EFormat Format() const; + + //! Returns serialized schema of arrow record batches + const std::string& GetArrowSchema() const; + + //! Returns bytes for binary data formats (arrow, etc.) + const std::vector& GetBytesData() const; + private: class TImpl; std::shared_ptr Impl_; diff --git a/src/api/protos/ydb_formats.proto b/src/api/protos/ydb_formats.proto index 287c440982..9fca88d653 100644 --- a/src/api/protos/ydb_formats.proto +++ b/src/api/protos/ydb_formats.proto @@ -25,3 +25,45 @@ message CsvSettings { bool header = 4; Quoting quoting = 5; } + +/** + * ArrowFormatSettings is settings for Ydb.ResultSet.Format.FORMAT_ARROW in Ydb.Query.ExecuteQueryRequest. + * It is used to configure compression for record batches in Ydb.ResultSet.data field. + */ +message ArrowFormatSettings { + message CompressionCodec { + enum Type { + // Unspecified mode, corresponds to TYPE_NONE + TYPE_UNSPECIFIED = 0; + + // Binary data without compression + TYPE_NONE = 1; + + // Zstandard compression + TYPE_ZSTD = 2; + + // LZ4 frame compression + TYPE_LZ4_FRAME = 3; + } + + // Type of the compression codec + Type type = 1; + + // Compression level for the codec. + // If is not specified, the default level of the codec type is used. + optional int32 level = 2; + } + + // Codec for compressing binary data in Ydb.ResultSet.data field + CompressionCodec compression_codec = 1; +} + +/** + * ArrowFormatMeta is a metadata for Ydb.ResultSet.Format.FORMAT_ARROW in Ydb.ResultSet. + * It is used to get the schema of the Arrow record batch. + */ +message ArrowFormatMeta { + // Schema of the arrow batch of the result. + // May be empty for custom Ydb.Query.SchemaInclusionMode + bytes schema = 1; +} diff --git a/src/api/protos/ydb_query.proto b/src/api/protos/ydb_query.proto index fa62727730..afc60032e6 100644 --- a/src/api/protos/ydb_query.proto +++ b/src/api/protos/ydb_query.proto @@ -9,6 +9,7 @@ import "google/protobuf/duration.proto"; import "src/api/protos/annotations/validation.proto"; import "src/api/protos/ydb_common.proto"; +import "src/api/protos/ydb_formats.proto"; import "src/api/protos/ydb_issue_message.proto"; import "src/api/protos/ydb_operation.proto"; import "src/api/protos/ydb_query_stats.proto"; @@ -155,6 +156,17 @@ enum StatsMode { STATS_MODE_PROFILE = 40; // Detailed execution stats including stats for individual tasks and channels } +enum SchemaInclusionMode { + // Unspecified mode, corresponds to SCHEMA_INCLUSION_MODE_ALWAYS. + SCHEMA_INCLUSION_MODE_UNSPECIFIED = 0; + + // Always include schema in every Ydb.ResultSet in the response stream. + SCHEMA_INCLUSION_MODE_ALWAYS = 1; + + // Include schema only in the first Ydb.ResultSet per result_set_index of the response stream. + SCHEMA_INCLUSION_MODE_FIRST_ONLY = 2; +} + message ExecuteQueryRequest { // Session identifier (required) string session_id = 1 [(Ydb.length).le = 1024]; @@ -182,6 +194,17 @@ message ExecuteQueryRequest { // When query statistics are enabled (stats_mode != STATS_MODE_NONE), by default statistics will be sent only once after query execution is finished. // In case when stats_period_ms is specified and is non-zero, query statistics will be additionally sent every stats_period_ms milliseconds beginning from the start of query execution. int64 stats_period_ms = 11 [(Ydb.value) = ">= 0"]; + + // Schema inclusion mode for the result sets. + // If is not specified, SchemaInclusionMode.SCHEMA_INCLUSION_MODE_ALWAYS is used. + SchemaInclusionMode schema_inclusion_mode = 12; + + // Format of the result sets. + // If is not specified, Ydb.ResultSet.Format.FORMAT_VALUE is used. + Ydb.ResultSet.Format result_set_format = 13; + + // Format settings, only used for Ydb.ResultSet.Format.FORMAT_ARROW + Ydb.Formats.ArrowFormatSettings arrow_format_settings = 14; } message ResultSetMeta { diff --git a/src/api/protos/ydb_value.proto b/src/api/protos/ydb_value.proto index 5f67a38b38..2e6e73d180 100644 --- a/src/api/protos/ydb_value.proto +++ b/src/api/protos/ydb_value.proto @@ -3,6 +3,8 @@ option cc_enable_arenas = true; import "google/protobuf/struct.proto"; +import "src/api/protos/ydb_formats.proto"; + package Ydb; option java_package = "com.yandex.ydb"; option java_outer_classname = "ValueProtos"; @@ -177,10 +179,30 @@ message Column { // Represents table-like structure with ordered set of rows and columns message ResultSet { - // Metadata of columns + enum Format { + // Unspecified, corresponds to FORMAT_VALUE + FORMAT_UNSPECIFIED = 0; + + // Used to get Ydb.Value as a result in Ydb.ResultSet.rows field + FORMAT_VALUE = 1; + + // Used to get Arrow record batch as a result in Ydb.ResultSet.data field + FORMAT_ARROW = 2; + } + + // Metadata of columns. + // May be empty for custom Ydb.Query.SchemaInclusionMode. repeated Column columns = 1; - // Rows of table + // Rows of table, are only filled for FORMAT_VALUE repeated Value rows = 2; // Flag indicates the result was truncated bool truncated = 3; + // Format of the result set rows + Format format = 4; + + // Metadata of the result set rows, only used for FORMAT_ARROW + Formats.ArrowFormatMeta arrow_format_meta = 5; + + // Binary data for compressible formats of rows + bytes data = 15; } diff --git a/src/client/arrow/accessor.cpp b/src/client/arrow/accessor.cpp new file mode 100644 index 0000000000..f337357eca --- /dev/null +++ b/src/client/arrow/accessor.cpp @@ -0,0 +1,17 @@ +#include + +namespace NYdb::inline V3 { + +TResultSet::EFormat TArrowAccessor::Format(const TResultSet& resultSet) { + return resultSet.Format(); +} + +const std::string& TArrowAccessor::GetArrowSchema(const TResultSet& resultSet) { + return resultSet.GetArrowSchema(); +} + +const std::vector& TArrowAccessor::GetArrowBatches(const TResultSet& resultSet) { + return resultSet.GetBytesData(); +} + +} // namespace NYdb diff --git a/src/client/query/impl/exec_query.cpp b/src/client/query/impl/exec_query.cpp index 852f5cf87e..4e694cb15d 100644 --- a/src/client/query/impl/exec_query.cpp +++ b/src/client/query/impl/exec_query.cpp @@ -150,6 +150,8 @@ struct TExecuteQueryBuffer : public TThrRefBase, TNonCopyable { std::vector ResultSets_; std::optional Stats_; std::optional Tx_; + std::vector ArrowSchemas_; + std::vector> BytesData_; void Next() { TPtr self(this); @@ -169,14 +171,22 @@ struct TExecuteQueryBuffer : public TThrRefBase, TNonCopyable { std::vector issues; std::vector resultProtos; std::optional tx; + std::vector arrowSchemas; + std::vector> bytesData; std::swap(self->Issues_, issues); std::swap(self->ResultSets_, resultProtos); std::swap(self->Tx_, tx); + std::swap(self->ArrowSchemas_, arrowSchemas); + std::swap(self->BytesData_, bytesData); std::vector resultSets; - for (auto& proto : resultProtos) { - resultSets.emplace_back(std::move(proto)); + for (size_t i = 0; i < resultProtos.size(); ++i) { + auto proto = std::move(resultProtos[i]); + auto arrowSchema = arrowSchemas.size() > i ? std::move(arrowSchemas[i]) : std::string(); + auto data = bytesData.size() > i ? std::move(bytesData[i]) : std::vector(); + + resultSets.emplace_back(std::move(proto), std::move(arrowSchema), std::move(data)); } self->Promise_.SetValue(TExecuteQueryResult( @@ -204,13 +214,19 @@ struct TExecuteQueryBuffer : public TThrRefBase, TNonCopyable { } auto& resultSet = self->ResultSets_[part.GetResultSetIndex()]; - if (resultSet.columns().empty()) { - resultSet.mutable_columns()->CopyFrom(inRsProto.columns()); - } + resultSet.set_format(inRsProto.format()); - resultSet.mutable_rows()->Reserve(resultSet.mutable_rows()->size() + inRsProto.rows_size()); - for (const auto& row : inRsProto.rows()) { - *resultSet.mutable_rows()->Add() = row; + switch (resultSet.format()) { + case Ydb::ResultSet::FORMAT_VALUE: { + self->CollectYdbValues(resultSet, inRsProto); + break; + } + case Ydb::ResultSet::FORMAT_ARROW: { + self->CollectArrowBytes(resultSet, inRs.MutableProto(), part.GetResultSetIndex()); + break; + } + default: + break; } } @@ -221,6 +237,40 @@ struct TExecuteQueryBuffer : public TThrRefBase, TNonCopyable { self->Next(); }); } + +private: + void CollectYdbValues(Ydb::ResultSet& resultSet, const Ydb::ResultSet& inRsProto) { + if (resultSet.columns().empty()) { + resultSet.mutable_columns()->CopyFrom(inRsProto.columns()); + } + + resultSet.mutable_rows()->Reserve(resultSet.mutable_rows()->size() + inRsProto.rows_size()); + for (const auto& row : inRsProto.rows()) { + *resultSet.mutable_rows()->Add() = row; + } + } + + void CollectArrowBytes(Ydb::ResultSet& resultSet, Ydb::ResultSet& mutableInRsProto, uint64_t index) { + if (resultSet.columns().empty()) { + resultSet.mutable_columns()->CopyFrom(mutableInRsProto.columns()); + } + + if (ArrowSchemas_.size() <= index) { + ArrowSchemas_.resize(index + 1); + BytesData_.resize(index + 1); + } + + auto& arrowSchema = ArrowSchemas_[index]; + auto& bytesData = BytesData_[index]; + + if (arrowSchema.empty()) { + arrowSchema = std::move(*mutableInRsProto.mutable_arrow_format_meta()->mutable_schema()); + } + + if (auto* data = mutableInRsProto.mutable_data(); data && !data->empty()) { + bytesData.emplace_back(std::move(*data)); + } + } }; class TExecQueryInternal { @@ -236,6 +286,8 @@ class TExecQueryInternal { request.set_pool_id(TStringType{settings.ResourcePool_}); request.mutable_query_content()->set_text(TStringType{query}); request.mutable_query_content()->set_syntax(::Ydb::Query::Syntax(settings.Syntax_)); + request.set_schema_inclusion_mode(::Ydb::Query::SchemaInclusionMode(settings.SchemaInclusionMode_)); + request.set_result_set_format(::Ydb::ResultSet::Format(settings.Format_)); if (session.has_value()) { request.set_session_id(TStringType{session->GetId()}); } else if ((std::holds_alternative(txControl.Tx_) && !txControl.CommitTx_) || @@ -256,6 +308,19 @@ class TExecQueryInternal { request.set_stats_period_ms(settings.StatsCollectPeriod_->count()); } + if (settings.ArrowFormatSettings_) { + auto formatSettings = request.mutable_arrow_format_settings(); + if (settings.ArrowFormatSettings_->CompressionCodec_) { + auto codec = formatSettings->mutable_compression_codec(); + auto type = settings.ArrowFormatSettings_->CompressionCodec_->Type_; + codec->set_type(::Ydb::Formats::ArrowFormatSettings::CompressionCodec::Type(type)); + + if (settings.ArrowFormatSettings_->CompressionCodec_->Level_) { + codec->set_level(*settings.ArrowFormatSettings_->CompressionCodec_->Level_); + } + } + } + if (txControl.HasTx()) { auto requestTxControl = request.mutable_tx_control(); requestTxControl->set_commit_tx(txControl.CommitTx_); diff --git a/src/client/result/result.cpp b/src/client/result/result.cpp index 97fd03c70d..0254606285 100644 --- a/src/client/result/result.cpp +++ b/src/client/result/result.cpp @@ -39,25 +39,55 @@ class TResultSet::TImpl { TImpl(const Ydb::ResultSet& proto) : ProtoResultSet_(proto) { - Init(); + Init(true); } TImpl(Ydb::ResultSet&& proto) : ProtoResultSet_(std::move(proto)) { - Init(); + Init(true); } - void Init() { + TImpl(const Ydb::ResultSet& proto, const std::string& arrowSchema, const std::vector& bytesData) + : ProtoResultSet_(proto) + , ArrowSchema_(arrowSchema) + , BytesData_(bytesData) + { + Init(false); + } + + TImpl(Ydb::ResultSet&& proto, std::string&& arrowSchema, std::vector&& bytesData) + : ProtoResultSet_(std::move(proto)) + , ArrowSchema_(std::move(arrowSchema)) + , BytesData_(std::move(bytesData)) + { + Init(false); + } + + void Init(bool extractArrowResult) { ColumnsMeta_.reserve(ProtoResultSet_.columns_size()); for (auto& meta : ProtoResultSet_.columns()) { ColumnsMeta_.push_back(TColumn(meta.name(), TType(meta.type()))); } + + auto format = static_cast(ProtoResultSet_.format()); + if (format == EFormat::Arrow && extractArrowResult) { + if (ProtoResultSet_.has_arrow_format_meta() && ArrowSchema_.empty()) { + ArrowSchema_ = std::move(*ProtoResultSet_.mutable_arrow_format_meta()->mutable_schema()); + } + + if (auto* data = ProtoResultSet_.mutable_data(); data && !data->empty()) { + BytesData_.push_back(std::move(*data)); + } + } } public: - const Ydb::ResultSet ProtoResultSet_; + Ydb::ResultSet ProtoResultSet_; std::vector ColumnsMeta_; + + std::string ArrowSchema_; + std::vector BytesData_; }; //////////////////////////////////////////////////////////////////////////////// @@ -68,6 +98,12 @@ TResultSet::TResultSet(const Ydb::ResultSet& proto) TResultSet::TResultSet(Ydb::ResultSet&& proto) : Impl_(new TResultSet::TImpl(std::move(proto))) {} +TResultSet::TResultSet(const Ydb::ResultSet& proto, const std::string& arrowSchema, const std::vector& bytesData) + : Impl_(new TResultSet::TImpl(proto, arrowSchema, bytesData)) {} + +TResultSet::TResultSet(Ydb::ResultSet&& proto, std::string&& arrowSchema, std::vector&& bytesData) + : Impl_(new TResultSet::TImpl(std::move(proto), std::move(arrowSchema), std::move(bytesData))) {} + size_t TResultSet::ColumnsCount() const { return Impl_->ColumnsMeta_.size(); } @@ -88,6 +124,22 @@ const Ydb::ResultSet& TResultSet::GetProto() const { return Impl_->ProtoResultSet_; } +Ydb::ResultSet& TResultSet::MutableProto() { + return Impl_->ProtoResultSet_; +} + +TResultSet::EFormat TResultSet::Format() const { + return static_cast(Impl_->ProtoResultSet_.format()); +} + +const std::string& TResultSet::GetArrowSchema() const { + return Impl_->ArrowSchema_; +} + +const std::vector& TResultSet::GetBytesData() const { + return Impl_->BytesData_; +} + //////////////////////////////////////////////////////////////////////////////// class TResultSetParser::TImpl { diff --git a/tests/integration/basic_example/main.cpp b/tests/integration/basic_example/main.cpp index ea3fff1a3f..72c4e07dfc 100644 --- a/tests/integration/basic_example/main.cpp +++ b/tests/integration/basic_example/main.cpp @@ -12,6 +12,8 @@ static void ValidateResultSet(const std::vector& columns, const std::vector>& values, const NYdb::TResultSet& rs) { Ydb::ResultSet protoResultSet; + protoResultSet.set_format(Ydb::ResultSet::FORMAT_VALUE); + for (const auto& column : columns) { auto* protoColumn = protoResultSet.add_columns(); protoColumn->set_name(column.Name); From b57354cd7074a8b3cea5cbe90b84963bb25db8a6 Mon Sep 17 00:00:00 2001 From: Daniil Timizhev Date: Wed, 27 Aug 2025 14:28:05 +0000 Subject: [PATCH 14/25] [C++ SDK] Fix compatibility for VALUE format (#22975) --- .github/last_commit.txt | 2 +- src/client/query/impl/exec_query.cpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 425801c2cc..0f678d92a7 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -d6599f50029614cb41dde51939369797b38400ba +bbae8f355a54e4426186fb29b94504c29da598c4 diff --git a/src/client/query/impl/exec_query.cpp b/src/client/query/impl/exec_query.cpp index 4e694cb15d..8f28a8f3fa 100644 --- a/src/client/query/impl/exec_query.cpp +++ b/src/client/query/impl/exec_query.cpp @@ -217,6 +217,7 @@ struct TExecuteQueryBuffer : public TThrRefBase, TNonCopyable { resultSet.set_format(inRsProto.format()); switch (resultSet.format()) { + case Ydb::ResultSet::FORMAT_UNSPECIFIED: case Ydb::ResultSet::FORMAT_VALUE: { self->CollectYdbValues(resultSet, inRsProto); break; From 7d6e12761987a47d152d5eb12ec9b15a3406c384 Mon Sep 17 00:00:00 2001 From: Tony-Romanov <150126326+Tony-Romanov@users.noreply.github.com> Date: Wed, 27 Aug 2025 14:28:11 +0000 Subject: [PATCH 15/25] Add external_data_channels_count creatte table feature. (#21151) --- .github/last_commit.txt | 2 +- include/ydb-cpp-sdk/client/table/table.h | 12 ++++++++++++ src/api/protos/ydb_table.proto | 4 ++++ src/client/table/table.cpp | 13 +++++++++++++ 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 0f678d92a7..156abdc093 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -bbae8f355a54e4426186fb29b94504c29da598c4 +7b49d48c39e2da6d713e8b832b440c62366e830f diff --git a/include/ydb-cpp-sdk/client/table/table.h b/include/ydb-cpp-sdk/client/table/table.h index f85d62c76f..f27de01b8d 100644 --- a/include/ydb-cpp-sdk/client/table/table.h +++ b/include/ydb-cpp-sdk/client/table/table.h @@ -635,6 +635,7 @@ class TStorageSettings { std::optional GetTabletCommitLog1() const; std::optional GetExternal() const; std::optional GetStoreExternalBlobs() const; + std::optional GetExternalDataChannelsCount() const; private: class TImpl; @@ -790,6 +791,7 @@ class TStorageSettingsBuilder { TStorageSettingsBuilder& SetTabletCommitLog1(const std::string& media); TStorageSettingsBuilder& SetExternal(const std::string& media); TStorageSettingsBuilder& SetStoreExternalBlobs(bool enabled); + TStorageSettingsBuilder& SetExternalDataChannelsCount(uint32_t count); TStorageSettings Build() const; @@ -864,6 +866,11 @@ class TTableStorageSettingsBuilder { return *this; } + TTableStorageSettingsBuilder& SetExternalDataChannelsCount(uint32_t count) { + Builder_.SetExternalDataChannelsCount(count); + return *this; + } + TTableBuilder& EndStorageSettings(); private: @@ -1483,6 +1490,11 @@ class TAlterStorageSettingsBuilder { return *this; } + TAlterStorageSettingsBuilder& SetExternalDataChannelsCount(uint32_t count) { + Builder_.SetExternalDataChannelsCount(count); + return *this; + } + TAlterTableSettings& EndAlterStorageSettings(); private: diff --git a/src/api/protos/ydb_table.proto b/src/api/protos/ydb_table.proto index 4c6bd864ec..42bba6085a 100644 --- a/src/api/protos/ydb_table.proto +++ b/src/api/protos/ydb_table.proto @@ -539,6 +539,10 @@ message StorageSettings { // * Table cannot be copied or backed up when this is enabled // * This feature cannot be disabled once enabled for a table Ydb.FeatureFlag.Status store_external_blobs = 5; + + // This works when store_external_blobs is enabled + // and specifies the number of data channels to store "external blobs" + optional uint32 external_data_channels_count = 6; } message ColumnFamily { diff --git a/src/client/table/table.cpp b/src/client/table/table.cpp index 7be5bd85fc..97b0f282dd 100644 --- a/src/client/table/table.cpp +++ b/src/client/table/table.cpp @@ -102,6 +102,14 @@ std::optional TStorageSettings::GetStoreExternalBlobs() const { } } +std::optional TStorageSettings::GetExternalDataChannelsCount() const { + if (GetProto().has_external_data_channels_count()) { + return GetProto().external_data_channels_count(); + } else { + return { }; + } +} + //////////////////////////////////////////////////////////////////////////////// class TColumnFamilyDescription::TImpl { @@ -995,6 +1003,11 @@ TStorageSettingsBuilder& TStorageSettingsBuilder::SetStoreExternalBlobs(bool ena return *this; } +TStorageSettingsBuilder& TStorageSettingsBuilder::SetExternalDataChannelsCount(uint32_t count) { + Impl_->Proto.set_external_data_channels_count(count); + return *this; +} + TStorageSettings TStorageSettingsBuilder::Build() const { return TStorageSettings(Impl_->Proto); } From 47912d2ad5ed1155fcaefcb67ded991b78247ea0 Mon Sep 17 00:00:00 2001 From: vporyadke Date: Wed, 27 Aug 2025 14:28:17 +0000 Subject: [PATCH 16/25] 2dc healthcheck updates: smarter merge + updates for arbiter (#23035) --- .github/last_commit.txt | 2 +- src/api/protos/ydb_monitoring.proto | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 156abdc093..21ab72188b 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -7b49d48c39e2da6d713e8b832b440c62366e830f +3049fabbec2cb2afb682aaf14c24fb5a57915366 diff --git a/src/api/protos/ydb_monitoring.proto b/src/api/protos/ydb_monitoring.proto index 704c9ce9ac..4609df2acb 100644 --- a/src/api/protos/ydb_monitoring.proto +++ b/src/api/protos/ydb_monitoring.proto @@ -127,6 +127,7 @@ message LocationNode { uint32 id = 1; string host = 2; uint32 port = 3; + LocationBridgePile pile = 4; } message LocationStoragePDisk { From c2fd7a39616ae974faa9635acf6e41c420da4cd8 Mon Sep 17 00:00:00 2001 From: Bulat Date: Wed, 27 Aug 2025 14:28:23 +0000 Subject: [PATCH 17/25] Refactor pile name in discovery response (#23152) --- .github/last_commit.txt | 2 +- src/api/protos/ydb_discovery.proto | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 21ab72188b..9b2ad4b087 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -3049fabbec2cb2afb682aaf14c24fb5a57915366 +c36ca71a188b4ec0f9246fed536b7e302cb12c2b diff --git a/src/api/protos/ydb_discovery.proto b/src/api/protos/ydb_discovery.proto index 003e43cec2..ea6580f85a 100644 --- a/src/api/protos/ydb_discovery.proto +++ b/src/api/protos/ydb_discovery.proto @@ -36,16 +36,14 @@ message EndpointInfo { // not this specific node hostname. string ssl_target_name_override = 10; - // EXPERIMENTAL // Optional name of the bridge pile this endpoint belongs to. - optional string bridge_pile_name = 11; + string bridge_pile_name = 11; } message ListEndpointsResult { repeated EndpointInfo endpoints = 1; string self_location = 2; - // EXPERIMENTAL // List of bridge pile states. // This field is empty if cluster is not in bridge mode. repeated Ydb.Bridge.PileState pile_states = 3; From eb971039be6d3480765340bb0fc55d14afe02b76 Mon Sep 17 00:00:00 2001 From: mregrock Date: Wed, 27 Aug 2025 14:28:29 +0000 Subject: [PATCH 18/25] Rename PROMOTE to PROMOTED (#23223) --- .github/last_commit.txt | 2 +- include/ydb-cpp-sdk/client/discovery/discovery.h | 2 +- src/api/protos/ydb_bridge_common.proto | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 9b2ad4b087..5d3c646ecb 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -c36ca71a188b4ec0f9246fed536b7e302cb12c2b +d9db1625470ae6eb1ece15686abff9a449a32df1 diff --git a/include/ydb-cpp-sdk/client/discovery/discovery.h b/include/ydb-cpp-sdk/client/discovery/discovery.h index 41c7c8736c..dcb5ffa778 100644 --- a/include/ydb-cpp-sdk/client/discovery/discovery.h +++ b/include/ydb-cpp-sdk/client/discovery/discovery.h @@ -69,7 +69,7 @@ struct TPileState { enum EState { UNSPECIFIED = 0 /* "unspecified" */, PRIMARY = 1 /* "primary" */, - PROMOTE = 2 /* "promote" */, + PROMOTED = 2 /* "promoted" */, SYNCHRONIZED = 3 /* "synchronized" */, NOT_SYNCHRONIZED = 4 /* "not_synchronized" */, SUSPENDED = 5 /* "suspended" */, diff --git a/src/api/protos/ydb_bridge_common.proto b/src/api/protos/ydb_bridge_common.proto index 30cd2852fd..1b19050680 100644 --- a/src/api/protos/ydb_bridge_common.proto +++ b/src/api/protos/ydb_bridge_common.proto @@ -13,7 +13,7 @@ message PileState { enum State { UNSPECIFIED = 0; PRIMARY = 1; // pile is primary - PROMOTE = 2; // pile is being promoted to primary + PROMOTED = 2; // pile is being promoted to primary SYNCHRONIZED = 3; // fully synchronized with primary, may be promoted NOT_SYNCHRONIZED = 4; // not synchronized with primary, cannot be promoted SUSPENDED = 5; // pile is gracefully taken down, preparing to be disconnected From 536ee9d7b7b97755d4e4f0939011d453cf935fd6 Mon Sep 17 00:00:00 2001 From: Bulat Date: Wed, 27 Aug 2025 14:28:35 +0000 Subject: [PATCH 19/25] [C++ SDK] Fixed tsan test fails (#23374) --- .github/last_commit.txt | 2 +- .../credentials/oauth2_token_exchange/credentials.cpp | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 5d3c646ecb..377be5fe14 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -d9db1625470ae6eb1ece15686abff9a449a32df1 +4ca2eda9f5b471a2aff4831a787bfd96eef54d68 diff --git a/src/client/types/credentials/oauth2_token_exchange/credentials.cpp b/src/client/types/credentials/oauth2_token_exchange/credentials.cpp index 081b52a0dc..31d9225709 100644 --- a/src/client/types/credentials/oauth2_token_exchange/credentials.cpp +++ b/src/client/types/credentials/oauth2_token_exchange/credentials.cpp @@ -443,18 +443,18 @@ class TOauth2TokenExchangeProviderImpl: public std::enable_shared_from_this TokenIsRefreshing = false; mutable std::thread RefreshTokenThread; mutable std::string Token; mutable TInstant TokenDeadline; From 586a5213c4f9f2fd305c6b34e6a9e17194aaf602 Mon Sep 17 00:00:00 2001 From: Bulat Date: Wed, 27 Aug 2025 14:28:41 +0000 Subject: [PATCH 20/25] [C++ SDK] Improved wide date types (#23153) --- .github/last_commit.txt | 2 +- examples/time/main.cpp | 62 +++++++ include/ydb-cpp-sdk/client/value/value.h | 38 ++-- src/client/value/value.cpp | 216 +++++++++++------------ 4 files changed, 193 insertions(+), 125 deletions(-) create mode 100644 examples/time/main.cpp diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 377be5fe14..a7fd72b1c2 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -4ca2eda9f5b471a2aff4831a787bfd96eef54d68 +f6e1fa8806a4d14c1b414586a42f44f976608122 diff --git a/examples/time/main.cpp b/examples/time/main.cpp new file mode 100644 index 0000000000..d1b3b707b8 --- /dev/null +++ b/examples/time/main.cpp @@ -0,0 +1,62 @@ +#include +#include + +#include + + +void TimeExample(const std::string& endpoint, const std::string& database) { + auto driverConfig = NYdb::CreateFromEnvironment(endpoint + "/?database=" + database); + NYdb::TDriver driver(driverConfig); + NYdb::NQuery::TQueryClient client(driver); + + NYdb::NStatusHelpers::ThrowOnError(client.RetryQuerySync([](NYdb::NQuery::TSession session) -> NYdb::TStatus { + std::string query = R"( + $ts1 = DateTime::MakeTimestamp64(Timestamp64("2019-05-12T15:30:18.123456Z")); + $ts2 = DateTime::MakeTimestamp64(Timestamp64("2019-05-12T15:30:19.012345Z")); + SELECT $ts1 as ts1, $ts2 as ts2, $ts2 - $ts1 as interval + )"; + + auto result = session.ExecuteQuery(query, NYdb::NQuery::TTxControl::NoTx()).GetValueSync(); + if (!result.IsSuccess()) { + return result; + } + + NYdb::TResultSetParser parser(result.GetResultSet(0)); + while (parser.TryNextRow()) { + auto ts1 = parser.ColumnParser("ts1").GetTimestamp64(); + auto ts2 = parser.ColumnParser("ts2").GetTimestamp64(); + auto interval = parser.ColumnParser("interval").GetInterval64(); + + std::cout << "ts1: " << ts1 << std::endl; + std::cout << "ts2: " << ts2 << std::endl; + std::cout << "interval: " << std::chrono::duration_cast(interval) << std::endl; + + std::cout << "ts1 + interval (" << ts1 + interval << ") = ts2 (" << ts2 << ")" << std::endl; + } + return result; + })); + + driver.Stop(true); +} + +int main(int argc, char** argv) { + NLastGetopt::TOpts opts = NLastGetopt::TOpts::Default(); + + std::string endpoint; + std::string database; + + opts.AddLongOption('e', "endpoint", "YDB endpoint").Required().RequiredArgument("HOST:PORT").StoreResult(&endpoint); + opts.AddLongOption('d', "database", "YDB database").Required().RequiredArgument("DATABASE").StoreResult(&database); + + opts.SetFreeArgsMin(0); + + NLastGetopt::TOptsParseResult result(&opts, argc, argv); + + try { + TimeExample(endpoint, database); + } catch (const std::exception& e) { + std::cerr << "Execution failed: " << e.what() << std::endl; + } + + return 0; +} diff --git a/include/ydb-cpp-sdk/client/value/value.h b/include/ydb-cpp-sdk/client/value/value.h index 5fa0c34304..7ac1b9f250 100644 --- a/include/ydb-cpp-sdk/client/value/value.h +++ b/include/ydb-cpp-sdk/client/value/value.h @@ -296,6 +296,12 @@ class TValue { std::shared_ptr Impl_; }; +//! Wide types are used to represent YDB date types (Date32, Datetime64, Timestamp64, Interval64). +//! They are used to avoid overflows when converting from YDB types to C++ types. +using TWideDays = std::chrono::duration>; +using TWideSeconds = std::chrono::duration>; +using TWideMicroseconds = std::chrono::duration; + class TValueParser : public TMoveOnly { friend class TResultSetParser; public: @@ -322,10 +328,10 @@ class TValueParser : public TMoveOnly { TInstant GetDatetime() const; TInstant GetTimestamp() const; int64_t GetInterval() const; - int32_t GetDate32() const; - int64_t GetDatetime64() const; - int64_t GetTimestamp64() const; - int64_t GetInterval64() const; + std::chrono::sys_time GetDate32() const; + std::chrono::sys_time GetDatetime64() const; + std::chrono::sys_time GetTimestamp64() const; + TWideMicroseconds GetInterval64() const; const std::string& GetTzDate() const; const std::string& GetTzDatetime() const; const std::string& GetTzTimestamp() const; @@ -354,10 +360,10 @@ class TValueParser : public TMoveOnly { std::optional GetOptionalDatetime() const; std::optional GetOptionalTimestamp() const; std::optional GetOptionalInterval() const; - std::optional GetOptionalDate32() const; - std::optional GetOptionalDatetime64() const; - std::optional GetOptionalTimestamp64() const; - std::optional GetOptionalInterval64() const; + std::optional> GetOptionalDate32() const; + std::optional> GetOptionalDatetime64() const; + std::optional> GetOptionalTimestamp64() const; + std::optional GetOptionalInterval64() const; std::optional GetOptionalTzDate() const; std::optional GetOptionalTzDatetime() const; std::optional GetOptionalTzTimestamp() const; @@ -448,10 +454,10 @@ class TValueBuilderBase : public TMoveOnly { TDerived& Uuid(const TUuidValue& value); TDerived& JsonDocument(const std::string& value); TDerived& DyNumber(const std::string& value); - TDerived& Date32(const int32_t value); - TDerived& Datetime64(const int64_t value); - TDerived& Timestamp64(const int64_t value); - TDerived& Interval64(const int64_t value); + TDerived& Date32(const std::chrono::sys_time& value); + TDerived& Datetime64(const std::chrono::sys_time& value); + TDerived& Timestamp64(const std::chrono::sys_time& value); + TDerived& Interval64(const TWideMicroseconds& value); TDerived& OptionalBool(const std::optional& value); TDerived& OptionalInt8(const std::optional& value); @@ -478,10 +484,10 @@ class TValueBuilderBase : public TMoveOnly { TDerived& OptionalUuid(const std::optional& value); TDerived& OptionalJsonDocument(const std::optional& value); TDerived& OptionalDyNumber(const std::optional& value); - TDerived& OptionalDate32(const std::optional& value); - TDerived& OptionalDatetime64(const std::optional& value); - TDerived& OptionalTimestamp64(const std::optional& value); - TDerived& OptionalInterval64(const std::optional& value); + TDerived& OptionalDate32(const std::optional>& value); + TDerived& OptionalDatetime64(const std::optional>& value); + TDerived& OptionalTimestamp64(const std::optional>& value); + TDerived& OptionalInterval64(const std::optional& value); // Optional TDerived& BeginOptional(); diff --git a/src/client/value/value.cpp b/src/client/value/value.cpp index 854c7f0292..2377d1dccf 100644 --- a/src/client/value/value.cpp +++ b/src/client/value/value.cpp @@ -1150,32 +1150,32 @@ class TValueParser::TImpl { return GetProto().bool_value(); } - i8 GetInt8() const { + int8_t GetInt8() const { CheckPrimitive(NYdb::EPrimitiveType::Int8); return GetProto().int32_value(); } - ui8 GetUint8() const { + uint8_t GetUint8() const { CheckPrimitive(NYdb::EPrimitiveType::Uint8); return GetProto().uint32_value(); } - i16 GetInt16() const { + int16_t GetInt16() const { CheckPrimitive(NYdb::EPrimitiveType::Int16); return GetProto().int32_value(); } - ui16 GetUint16() const { + uint16_t GetUint16() const { CheckPrimitive(NYdb::EPrimitiveType::Uint16); return GetProto().uint32_value(); } - i32 GetInt32() const { + int32_t GetInt32() const { CheckPrimitive(NYdb::EPrimitiveType::Int32); return GetProto().int32_value(); } - ui32 GetUint32() const { + uint32_t GetUint32() const { CheckPrimitive(NYdb::EPrimitiveType::Uint32); return GetProto().uint32_value(); } @@ -1220,24 +1220,24 @@ class TValueParser::TImpl { return GetProto().int64_value(); } - i32 GetDate32() const { + std::chrono::sys_time GetDate32() const { CheckPrimitive(NYdb::EPrimitiveType::Date32); - return GetProto().int32_value(); + return std::chrono::sys_time(TWideDays(GetProto().int32_value())); } - int64_t GetDatetime64() const { + std::chrono::sys_time GetDatetime64() const { CheckPrimitive(NYdb::EPrimitiveType::Datetime64); - return GetProto().int64_value(); + return std::chrono::sys_time(TWideSeconds(GetProto().int64_value())); } - int64_t GetTimestamp64() const { + std::chrono::sys_time GetTimestamp64() const { CheckPrimitive(NYdb::EPrimitiveType::Timestamp64); - return GetProto().int64_value(); + return std::chrono::sys_time(TWideMicroseconds(GetProto().int64_value())); } - int64_t GetInterval64() const { + TWideMicroseconds GetInterval64() const { CheckPrimitive(NYdb::EPrimitiveType::Interval64); - return GetProto().int64_value(); + return TWideMicroseconds(GetProto().int64_value()); } const std::string& GetTzDate() const { @@ -1692,27 +1692,27 @@ bool TValueParser::GetBool() const { return Impl_->GetBool(); } -i8 TValueParser::GetInt8() const { +int8_t TValueParser::GetInt8() const { return Impl_->GetInt8(); } -ui8 TValueParser::GetUint8() const { +uint8_t TValueParser::GetUint8() const { return Impl_->GetUint8(); } -i16 TValueParser::GetInt16() const { +int16_t TValueParser::GetInt16() const { return Impl_->GetInt16(); } -ui16 TValueParser::GetUint16() const { +uint16_t TValueParser::GetUint16() const { return Impl_->GetUint16(); } -i32 TValueParser::GetInt32() const { +int32_t TValueParser::GetInt32() const { return Impl_->GetInt32(); } -ui32 TValueParser::GetUint32() const { +uint32_t TValueParser::GetUint32() const { return Impl_->GetUint32(); } @@ -1748,19 +1748,19 @@ int64_t TValueParser::GetInterval() const { return Impl_->GetInterval(); } -i32 TValueParser::GetDate32() const { +std::chrono::sys_time TValueParser::GetDate32() const { return Impl_->GetDate32(); } -int64_t TValueParser::GetDatetime64() const { +std::chrono::sys_time TValueParser::GetDatetime64() const { return Impl_->GetDatetime64(); } -int64_t TValueParser::GetTimestamp64() const { +std::chrono::sys_time TValueParser::GetTimestamp64() const { return Impl_->GetTimestamp64(); } -int64_t TValueParser::GetInterval64() const { +TWideMicroseconds TValueParser::GetInterval64() const { return Impl_->GetInterval64(); } @@ -1824,28 +1824,28 @@ std::optional TValueParser::GetOptionalBool() const { RET_OPT_VALUE(bool, Bool); } -std::optional TValueParser::GetOptionalInt8() const { - RET_OPT_VALUE(i8, Int8); +std::optional TValueParser::GetOptionalInt8() const { + RET_OPT_VALUE(int8_t, Int8); } -std::optional TValueParser::GetOptionalUint8() const { - RET_OPT_VALUE(ui8, Uint8); +std::optional TValueParser::GetOptionalUint8() const { + RET_OPT_VALUE(uint8_t, Uint8); } -std::optional TValueParser::GetOptionalInt16() const { - RET_OPT_VALUE(i16, Int16); +std::optional TValueParser::GetOptionalInt16() const { + RET_OPT_VALUE(int16_t, Int16); } -std::optional TValueParser::GetOptionalUint16() const { - RET_OPT_VALUE(ui16, Uint16); +std::optional TValueParser::GetOptionalUint16() const { + RET_OPT_VALUE(uint16_t, Uint16); } -std::optional TValueParser::GetOptionalInt32() const { - RET_OPT_VALUE(i32, Int32); +std::optional TValueParser::GetOptionalInt32() const { + RET_OPT_VALUE(int32_t, Int32); } -std::optional TValueParser::GetOptionalUint32() const { - RET_OPT_VALUE(ui32, Uint32); +std::optional TValueParser::GetOptionalUint32() const { + RET_OPT_VALUE(uint32_t, Uint32); } std::optional TValueParser::GetOptionalInt64() const { @@ -1880,20 +1880,20 @@ std::optional TValueParser::GetOptionalInterval() const { RET_OPT_VALUE(int64_t, Interval); } -std::optional TValueParser::GetOptionalDate32() const { - RET_OPT_VALUE(int64_t, Date32); +std::optional> TValueParser::GetOptionalDate32() const { + RET_OPT_VALUE(std::chrono::sys_time, Date32); } -std::optional TValueParser::GetOptionalDatetime64() const { - RET_OPT_VALUE(int64_t, Datetime64); +std::optional> TValueParser::GetOptionalDatetime64() const { + RET_OPT_VALUE(std::chrono::sys_time, Datetime64); } -std::optional TValueParser::GetOptionalTimestamp64() const { - RET_OPT_VALUE(int64_t, Timestamp64); +std::optional> TValueParser::GetOptionalTimestamp64() const { + RET_OPT_VALUE(std::chrono::sys_time, Timestamp64); } -std::optional TValueParser::GetOptionalInterval64() const { - RET_OPT_VALUE(int64_t, Interval64); +std::optional TValueParser::GetOptionalInterval64() const { + RET_OPT_VALUE(TWideMicroseconds, Interval64); } std::optional TValueParser::GetOptionalTzDate() const { @@ -2118,32 +2118,32 @@ class TValueBuilderImpl { GetValue().set_bool_value(value); } - void Int8(i8 value) { + void Int8(int8_t value) { FillPrimitiveType(EPrimitiveType::Int8); GetValue().set_int32_value(value); } - void Uint8(ui8 value) { + void Uint8(uint8_t value) { FillPrimitiveType(EPrimitiveType::Uint8); GetValue().set_uint32_value(value); } - void Int16(i16 value) { + void Int16(int16_t value) { FillPrimitiveType(EPrimitiveType::Int16); GetValue().set_int32_value(value); } - void Uint16(ui16 value) { + void Uint16(uint16_t value) { FillPrimitiveType(EPrimitiveType::Uint16); GetValue().set_uint32_value(value); } - void Int32(i32 value) { + void Int32(int32_t value) { FillPrimitiveType(EPrimitiveType::Int32); GetValue().set_int32_value(value); } - void Uint32(ui32 value) { + void Uint32(uint32_t value) { FillPrimitiveType(EPrimitiveType::Uint32); GetValue().set_uint32_value(value); } @@ -2188,24 +2188,24 @@ class TValueBuilderImpl { GetValue().set_int64_value(value); } - void Date32(const i32 value) { + void Date32(const std::chrono::sys_time& value) { FillPrimitiveType(EPrimitiveType::Date32); - GetValue().set_int32_value(value); + GetValue().set_int32_value(value.time_since_epoch().count()); } - void Datetime64(const int64_t value) { + void Datetime64(const std::chrono::sys_time& value) { FillPrimitiveType(EPrimitiveType::Datetime64); - GetValue().set_int64_value(value); + GetValue().set_int64_value(value.time_since_epoch().count()); } - void Timestamp64(const int64_t value) { + void Timestamp64(const std::chrono::sys_time& value) { FillPrimitiveType(EPrimitiveType::Timestamp64); - GetValue().set_int64_value(value); + GetValue().set_int64_value(value.time_since_epoch().count()); } - void Interval64(const int64_t value) { + void Interval64(const TWideMicroseconds& value) { FillPrimitiveType(EPrimitiveType::Interval64); - GetValue().set_int64_value(value); + GetValue().set_int64_value(value.count()); } void TzDate(const std::string& value) { @@ -2870,37 +2870,37 @@ TDerived& TValueBuilderBase::Bool(bool value) { } template -TDerived& TValueBuilderBase::Int8(i8 value) { +TDerived& TValueBuilderBase::Int8(int8_t value) { Impl_->Int8(value); return static_cast(*this); } template -TDerived& TValueBuilderBase::Uint8(ui8 value) { +TDerived& TValueBuilderBase::Uint8(uint8_t value) { Impl_->Uint8(value); return static_cast(*this); } template -TDerived& TValueBuilderBase::Int16(i16 value) { +TDerived& TValueBuilderBase::Int16(int16_t value) { Impl_->Int16(value); return static_cast(*this); } template -TDerived& TValueBuilderBase::Uint16(ui16 value) { +TDerived& TValueBuilderBase::Uint16(uint16_t value) { Impl_->Uint16(value); return static_cast(*this); } template -TDerived& TValueBuilderBase::Int32(i32 value) { +TDerived& TValueBuilderBase::Int32(int32_t value) { Impl_->Int32(value); return static_cast(*this); } template -TDerived& TValueBuilderBase::Uint32(ui32 value) { +TDerived& TValueBuilderBase::Uint32(uint32_t value) { Impl_->Uint32(value); return static_cast(*this); } @@ -2954,25 +2954,25 @@ TDerived& TValueBuilderBase::Interval(int64_t value) { } template -TDerived& TValueBuilderBase::Date32(const i32 value) { +TDerived& TValueBuilderBase::Date32(const std::chrono::sys_time& value) { Impl_->Date32(value); return static_cast(*this); } template -TDerived& TValueBuilderBase::Datetime64(const int64_t value) { +TDerived& TValueBuilderBase::Datetime64(const std::chrono::sys_time& value) { Impl_->Datetime64(value); return static_cast(*this); } template -TDerived& TValueBuilderBase::Timestamp64(const int64_t value) { +TDerived& TValueBuilderBase::Timestamp64(const std::chrono::sys_time& value) { Impl_->Timestamp64(value); return static_cast(*this); } template -TDerived& TValueBuilderBase::Interval64(const int64_t value) { +TDerived& TValueBuilderBase::Interval64(const TWideMicroseconds& value) { Impl_->Interval64(value); return static_cast(*this); } @@ -3049,7 +3049,7 @@ TDerived& TValueBuilderBase::Pg(const TPgValue& value) { return static_cast(*this); } -#define SET_OPT_VALUE_MAYBE(Name) \ +#define SET_OPT_VALUE_FROM_OPTIONAL(Name) \ if (value) { \ Impl_->BeginOptional(); \ Impl_->Name(*value); \ @@ -3067,147 +3067,147 @@ TDerived& TValueBuilderBase::Pg(const TPgValue& value) { template TDerived& TValueBuilderBase::OptionalBool(const std::optional& value) { - SET_OPT_VALUE_MAYBE(Bool); + SET_OPT_VALUE_FROM_OPTIONAL(Bool); } template -TDerived& TValueBuilderBase::OptionalInt8(const std::optional& value) { - SET_OPT_VALUE_MAYBE(Int8); +TDerived& TValueBuilderBase::OptionalInt8(const std::optional& value) { + SET_OPT_VALUE_FROM_OPTIONAL(Int8); } template -TDerived& TValueBuilderBase::OptionalUint8(const std::optional& value) { - SET_OPT_VALUE_MAYBE(Uint8); +TDerived& TValueBuilderBase::OptionalUint8(const std::optional& value) { + SET_OPT_VALUE_FROM_OPTIONAL(Uint8); } template -TDerived& TValueBuilderBase::OptionalInt16(const std::optional& value) { - SET_OPT_VALUE_MAYBE(Int16); +TDerived& TValueBuilderBase::OptionalInt16(const std::optional& value) { + SET_OPT_VALUE_FROM_OPTIONAL(Int16); } template -TDerived& TValueBuilderBase::OptionalUint16(const std::optional& value) { - SET_OPT_VALUE_MAYBE(Uint16); +TDerived& TValueBuilderBase::OptionalUint16(const std::optional& value) { + SET_OPT_VALUE_FROM_OPTIONAL(Uint16); } template -TDerived& TValueBuilderBase::OptionalInt32(const std::optional& value) { - SET_OPT_VALUE_MAYBE(Int32); +TDerived& TValueBuilderBase::OptionalInt32(const std::optional& value) { + SET_OPT_VALUE_FROM_OPTIONAL(Int32); } template -TDerived& TValueBuilderBase::OptionalUint32(const std::optional& value) { - SET_OPT_VALUE_MAYBE(Uint32); +TDerived& TValueBuilderBase::OptionalUint32(const std::optional& value) { + SET_OPT_VALUE_FROM_OPTIONAL(Uint32); } template TDerived& TValueBuilderBase::OptionalInt64(const std::optional& value) { - SET_OPT_VALUE_MAYBE(Int64); + SET_OPT_VALUE_FROM_OPTIONAL(Int64); } template TDerived& TValueBuilderBase::OptionalUint64(const std::optional& value) { - SET_OPT_VALUE_MAYBE(Uint64); + SET_OPT_VALUE_FROM_OPTIONAL(Uint64); } template TDerived& TValueBuilderBase::OptionalFloat(const std::optional& value) { - SET_OPT_VALUE_MAYBE(Float); + SET_OPT_VALUE_FROM_OPTIONAL(Float); } template TDerived& TValueBuilderBase::OptionalDouble(const std::optional& value) { - SET_OPT_VALUE_MAYBE(Double); + SET_OPT_VALUE_FROM_OPTIONAL(Double); } template TDerived& TValueBuilderBase::OptionalDate(const std::optional& value) { - SET_OPT_VALUE_MAYBE(Date); + SET_OPT_VALUE_FROM_OPTIONAL(Date); } template TDerived& TValueBuilderBase::OptionalDatetime(const std::optional& value) { - SET_OPT_VALUE_MAYBE(Datetime); + SET_OPT_VALUE_FROM_OPTIONAL(Datetime); } template TDerived& TValueBuilderBase::OptionalTimestamp(const std::optional& value) { - SET_OPT_VALUE_MAYBE(Timestamp); + SET_OPT_VALUE_FROM_OPTIONAL(Timestamp); } template TDerived& TValueBuilderBase::OptionalInterval(const std::optional& value) { - SET_OPT_VALUE_MAYBE(Interval); + SET_OPT_VALUE_FROM_OPTIONAL(Interval); } template -TDerived& TValueBuilderBase::OptionalDate32(const std::optional& value) { - SET_OPT_VALUE_MAYBE(Date32); +TDerived& TValueBuilderBase::OptionalDate32(const std::optional>& value) { + SET_OPT_VALUE_FROM_OPTIONAL(Date32); } template -TDerived& TValueBuilderBase::OptionalDatetime64(const std::optional& value) { - SET_OPT_VALUE_MAYBE(Datetime64); +TDerived& TValueBuilderBase::OptionalDatetime64(const std::optional>& value) { + SET_OPT_VALUE_FROM_OPTIONAL(Datetime64); } template -TDerived& TValueBuilderBase::OptionalTimestamp64(const std::optional& value) { - SET_OPT_VALUE_MAYBE(Timestamp64); +TDerived& TValueBuilderBase::OptionalTimestamp64(const std::optional>& value) { + SET_OPT_VALUE_FROM_OPTIONAL(Timestamp64); } template -TDerived& TValueBuilderBase::OptionalInterval64(const std::optional& value) { - SET_OPT_VALUE_MAYBE(Interval64); +TDerived& TValueBuilderBase::OptionalInterval64(const std::optional& value) { + SET_OPT_VALUE_FROM_OPTIONAL(Interval64); } template TDerived& TValueBuilderBase::OptionalTzDate(const std::optional& value) { - SET_OPT_VALUE_MAYBE(TzDate); + SET_OPT_VALUE_FROM_OPTIONAL(TzDate); } template TDerived& TValueBuilderBase::OptionalTzDatetime(const std::optional& value) { - SET_OPT_VALUE_MAYBE(TzDatetime); + SET_OPT_VALUE_FROM_OPTIONAL(TzDatetime); } template TDerived& TValueBuilderBase::OptionalTzTimestamp(const std::optional& value) { - SET_OPT_VALUE_MAYBE(TzTimestamp); + SET_OPT_VALUE_FROM_OPTIONAL(TzTimestamp); } template TDerived& TValueBuilderBase::OptionalString(const std::optional& value) { - SET_OPT_VALUE_MAYBE(String); + SET_OPT_VALUE_FROM_OPTIONAL(String); } template TDerived& TValueBuilderBase::OptionalUtf8(const std::optional& value) { - SET_OPT_VALUE_MAYBE(Utf8); + SET_OPT_VALUE_FROM_OPTIONAL(Utf8); } template TDerived& TValueBuilderBase::OptionalYson(const std::optional& value) { - SET_OPT_VALUE_MAYBE(Yson); + SET_OPT_VALUE_FROM_OPTIONAL(Yson); } template TDerived& TValueBuilderBase::OptionalJson(const std::optional& value) { - SET_OPT_VALUE_MAYBE(Json); + SET_OPT_VALUE_FROM_OPTIONAL(Json); } template TDerived& TValueBuilderBase::OptionalUuid(const std::optional& value) { - SET_OPT_VALUE_MAYBE(Uuid); + SET_OPT_VALUE_FROM_OPTIONAL(Uuid); } template TDerived& TValueBuilderBase::OptionalJsonDocument(const std::optional& value) { - SET_OPT_VALUE_MAYBE(JsonDocument); + SET_OPT_VALUE_FROM_OPTIONAL(JsonDocument); } template TDerived& TValueBuilderBase::OptionalDyNumber(const std::optional& value) { - SET_OPT_VALUE_MAYBE(DyNumber); + SET_OPT_VALUE_FROM_OPTIONAL(DyNumber); } From 043178a0ebaa01dbe34d59f7c4f9c10a0d4e4952 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Wed, 27 Aug 2025 14:28:47 +0000 Subject: [PATCH 21/25] Add restore long ops (#22710) --- .github/last_commit.txt | 2 +- .../library/operation_id/operation_id.h | 1 + src/api/protos/draft/ydb_backup.proto | 19 +++++++++++++++++++ src/client/operation/operation.cpp | 6 ++++++ src/library/operation_id/operation_id.cpp | 7 +++++++ .../operation_id/protos/operation_id.proto | 1 + 6 files changed, 35 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index a7fd72b1c2..6956e71615 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -f6e1fa8806a4d14c1b414586a42f44f976608122 +ec82fcb585c5439dafa7b6a9d3df8f8da9e353aa diff --git a/include/ydb-cpp-sdk/library/operation_id/operation_id.h b/include/ydb-cpp-sdk/library/operation_id/operation_id.h index cdcce79e39..0524ceccd8 100644 --- a/include/ydb-cpp-sdk/library/operation_id/operation_id.h +++ b/include/ydb-cpp-sdk/library/operation_id/operation_id.h @@ -27,6 +27,7 @@ class TOperationId { SCRIPT_EXECUTION = 9, SS_BG_TASKS = 10, INCREMENTAL_BACKUP = 11, + RESTORE = 12, }; struct TData { diff --git a/src/api/protos/draft/ydb_backup.proto b/src/api/protos/draft/ydb_backup.proto index ec7d7c8864..43dd1790fa 100644 --- a/src/api/protos/draft/ydb_backup.proto +++ b/src/api/protos/draft/ydb_backup.proto @@ -196,3 +196,22 @@ message IncrementalBackupMetadata { message IncrementalBackupResult { } + +message RestoreProgress { + enum Progress { + PROGRESS_UNSPECIFIED = 0; + PROGRESS_PREPARING = 1; + PROGRESS_TRANSFER_DATA = 2; + PROGRESS_DONE = 3; + PROGRESS_CANCELLATION = 4; + PROGRESS_CANCELLED = 5; + } +} + +message RestoreMetadata { + RestoreProgress.Progress progress = 1; + int32 progress_percent = 2 [(Ydb.value) = "[0; 100]"]; +} + +message RestoreResult { +} diff --git a/src/client/operation/operation.cpp b/src/client/operation/operation.cpp index a9ed76d202..737c484726 100644 --- a/src/client/operation/operation.cpp +++ b/src/client/operation/operation.cpp @@ -216,6 +216,12 @@ template <> TFuture> TOperationClient::List(ui64 pageSize, const std::string& pageToken) { return List("incbackup", pageSize, pageToken); } + +template TFuture TOperationClient::Get(const TOperation::TOperationId& id); +template <> +TFuture> TOperationClient::List(ui64 pageSize, const std::string& pageToken) { + return List("restore", pageSize, pageToken); +} #endif } // namespace NOperation diff --git a/src/library/operation_id/operation_id.cpp b/src/library/operation_id/operation_id.cpp index d9d667607d..ea7e2e1bf3 100644 --- a/src/library/operation_id/operation_id.cpp +++ b/src/library/operation_id/operation_id.cpp @@ -69,6 +69,9 @@ std::string ProtoToString(const Ydb::TOperationId& proto) { case Ydb::TOperationId::INCREMENTAL_BACKUP: res << "ydb://incbackup"; break; + case Ydb::TOperationId::RESTORE: + res << "ydb://restore"; + break; default: Y_ABORT_UNLESS(false, "unexpected kind"); } @@ -312,6 +315,10 @@ TOperationId::EKind ParseKind(const std::string_view value) { return TOperationId::INCREMENTAL_BACKUP; } + if (value.starts_with("restore")) { + return TOperationId::RESTORE; + } + return TOperationId::UNUSED; } diff --git a/src/library/operation_id/protos/operation_id.proto b/src/library/operation_id/protos/operation_id.proto index 5e8a2f6ff0..936feb2d93 100644 --- a/src/library/operation_id/protos/operation_id.proto +++ b/src/library/operation_id/protos/operation_id.proto @@ -16,6 +16,7 @@ message TOperationId { SCRIPT_EXECUTION = 9; SS_BG_TASKS = 10; INCREMENTAL_BACKUP = 11; + RESTORE = 12; } message TData { From 6653d26aa58f099873960c4b8ce3b1f30babf40a Mon Sep 17 00:00:00 2001 From: mregrock Date: Wed, 27 Aug 2025 14:28:54 +0000 Subject: [PATCH 22/25] Add generation in bridge list (#23511) --- .github/last_commit.txt | 2 +- src/api/protos/draft/ydb_bridge.proto | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 6956e71615..73639de5f9 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -ec82fcb585c5439dafa7b6a9d3df8f8da9e353aa +166732d832b4cf5e4222ec3785e15065cd327dce diff --git a/src/api/protos/draft/ydb_bridge.proto b/src/api/protos/draft/ydb_bridge.proto index 50c5daf692..eab69b1348 100644 --- a/src/api/protos/draft/ydb_bridge.proto +++ b/src/api/protos/draft/ydb_bridge.proto @@ -23,6 +23,8 @@ message GetClusterStateResponse { message GetClusterStateResult { // Current cluster state repeated PileState pile_states = 1; + // Generation of the cluster state + uint64 generation = 2; } message UpdateClusterStateRequest { From f4d2dc7813b5baca704a280533e33900bae48376 Mon Sep 17 00:00:00 2001 From: Bulat Date: Wed, 27 Aug 2025 14:29:00 +0000 Subject: [PATCH 23/25] [C++ SDK] Refactor IAM cred provider (#23411) --- .github/last_commit.txt | 2 +- .../client/iam/common/generic_provider.h | 90 ++++++++++--------- .../library/grpc_common}/constants.h | 4 +- src/client/iam/iam.cpp | 2 +- src/client/iam_private/common/iam.h | 32 ++++--- src/library/grpc/client/grpc_client_low.h | 1 - src/library/grpc/client/grpc_common.h | 6 +- 7 files changed, 75 insertions(+), 62 deletions(-) rename src/client/iam/common/iam.h => include/ydb-cpp-sdk/client/iam/common/generic_provider.h (71%) rename {src/library/grpc/common => include/ydb-cpp-sdk/library/grpc_common}/constants.h (69%) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 73639de5f9..72b20a13cc 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -166732d832b4cf5e4222ec3785e15065cd327dce +362cf04ca2b49a71004ea0558dbf5a041db0e0c6 diff --git a/src/client/iam/common/iam.h b/include/ydb-cpp-sdk/client/iam/common/generic_provider.h similarity index 71% rename from src/client/iam/common/iam.h rename to include/ydb-cpp-sdk/client/iam/common/generic_provider.h index 8ce8522cd4..c596fd6c75 100644 --- a/src/client/iam/common/iam.h +++ b/include/ydb-cpp-sdk/client/iam/common/generic_provider.h @@ -1,37 +1,36 @@ #pragma once -#include +#include "types.h" -#include +#include +#include #include #include #include +#include + namespace NYdb::inline V3 { constexpr TDuration BACKOFF_START = TDuration::MilliSeconds(50); constexpr TDuration BACKOFF_MAX = TDuration::Seconds(10); +// This file contains internal generic implementation of IAM credentials providers. +// DO NOT USE THIS CLASS DIRECTLY. Use specialized factory methods for specific cases. template class TGrpcIamCredentialsProvider : public ICredentialsProvider { protected: using TRequestFiller = std::function; - - using TSimpleRpc = - typename NYdbGrpc::TSimpleRequestProcessor< - typename TService::Stub, - TRequest, - TResponse>::TAsyncRequest; + using TAsyncInterface = typename TService::Stub::async_interface; + using TAsyncRpc = void (TAsyncInterface::*)(grpc::ClientContext*, const TRequest*, TResponse*, std::function); private: class TImpl : public std::enable_shared_from_this::TImpl> { public: - TImpl(const TIamEndpoint& iamEndpoint, const TRequestFiller& requestFiller, TSimpleRpc rpc) - : Client(std::make_unique()) - , Connection_(nullptr) - , Rpc_(rpc) + TImpl(const TIamEndpoint& iamEndpoint, const TRequestFiller& requestFiller, TAsyncRpc rpc) + : Rpc_(rpc) , Ticket_("") , NextTicketUpdate_(TInstant::Zero()) , IamEndpoint_(iamEndpoint) @@ -42,13 +41,22 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { , BackoffTimeout_(BACKOFF_START) , Lock_() { - NYdbGrpc::TGRpcClientConfig grpcConf; - grpcConf.Locator = IamEndpoint_.Endpoint; - grpcConf.EnableSsl = IamEndpoint_.EnableSsl; - if (!IamEndpoint_.CaCerts.empty()) { - grpcConf.SslCredentials.pem_root_certs = IamEndpoint_.CaCerts; + std::shared_ptr creds = nullptr; + if (IamEndpoint_.EnableSsl) { + grpc::SslCredentialsOptions opts; + opts.pem_root_certs = IamEndpoint_.CaCerts; + creds = grpc::SslCredentials(opts); + } else { + creds = grpc::InsecureChannelCredentials(); } - Connection_ = std::unique_ptr>(Client->CreateGRpcServiceConnection(grpcConf).release()); + + grpc::ChannelArguments args; + + args.SetMaxSendMessageSize(NGrpc::DEFAULT_GRPC_MESSAGE_SIZE_LIMIT); + args.SetMaxReceiveMessageSize(NGrpc::DEFAULT_GRPC_MESSAGE_SIZE_LIMIT); + + Channel_ = grpc::CreateCustomChannel(grpc::string{IamEndpoint_.Endpoint}, creds, args); + Stub_ = TService::NewStub(Channel_); } void UpdateTicket(bool sync = false) { @@ -61,12 +69,12 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { } auto resultPromise = NThreading::NewPromise(); + auto response = std::make_shared(); std::shared_ptr self = TGrpcIamCredentialsProvider::TImpl::shared_from_this(); - auto cb = [self, resultPromise, sync]( - NYdbGrpc::TGrpcStatus&& status, TResponse&& result) mutable { - self->ProcessIamResponse(std::move(status), std::move(result), sync); + auto cb = [self, sync, resultPromise, response] (grpc::Status status) mutable { + self->ProcessIamResponse(std::move(status), std::move(*response), sync); resultPromise.SetValue(); }; @@ -74,12 +82,15 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { RequestFiller_(req); - Connection_->template DoRequest( - std::move(req), - std::move(cb), - Rpc_, - { {}, {}, IamEndpoint_.RequestTimeout } - ); + grpc::ClientContext context; + + auto deadline = gpr_time_add( + gpr_now(GPR_CLOCK_MONOTONIC), + gpr_time_from_micros(IamEndpoint_.RequestTimeout.MicroSeconds(), GPR_TIMESPAN)); + + context.set_deadline(deadline); + + (Stub_->async()->*Rpc_)(&context, &req, response.get(), std::move(cb)); if (sync) { resultPromise.GetFuture().Wait(2 * IamEndpoint_.RequestTimeout); @@ -110,22 +121,19 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { } NeedStop_ = true; } - - Client.reset(); // Will trigger destroy } private: - void ProcessIamResponse(NYdbGrpc::TGrpcStatus&& status, TResponse&& result, bool sync) { - if (!status.Ok()) { + void ProcessIamResponse(grpc::Status&& status, TResponse&& result, bool sync) { + if (!status.ok()) { TDuration sleepDuration; { std::lock_guard guard(Lock_); LastRequestError_ = TStringBuilder() << "Last request error was at " << TInstant::Now() - << ". GrpcStatusCode: " << status.GRpcStatusCode - << " Message: \"" << status.Msg - << "\" internal: " << status.InternalError - << " iam-endpoint: \"" << IamEndpoint_.Endpoint << "\""; + << ". GrpcStatusCode: " << static_cast(status.error_code()) + << " Message: \"" << status.error_message() + << "\" iam-endpoint: \"" << IamEndpoint_.Endpoint << "\""; RequestInflight_ = false; sleepDuration = std::min(BackoffTimeout_, BACKOFF_MAX); @@ -152,9 +160,9 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { } private: - std::unique_ptr Client; - std::unique_ptr> Connection_; - TSimpleRpc Rpc_; + std::shared_ptr Channel_; + std::shared_ptr Stub_; + TAsyncRpc Rpc_; std::string Ticket_; TInstant NextTicketUpdate_; const TIamEndpoint IamEndpoint_; @@ -167,7 +175,7 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { }; public: - TGrpcIamCredentialsProvider(const TIamEndpoint& endpoint, const TRequestFiller& requestFiller, TSimpleRpc rpc) + TGrpcIamCredentialsProvider(const TIamEndpoint& endpoint, const TRequestFiller& requestFiller, TAsyncRpc rpc) : Impl_(std::make_shared(endpoint, requestFiller, rpc)) { Impl_->UpdateTicket(true); @@ -196,7 +204,7 @@ class TIamJwtCredentialsProvider : public TGrpcIamCredentialsProvider(params, [jwtParams = params.JwtParams](TRequest& req) { req.set_jwt(MakeSignedJwt(jwtParams)); - }, &TService::Stub::AsyncCreate) {} + }, &TService::Stub::async_interface::Create) {} }; template @@ -206,7 +214,7 @@ class TIamOAuthCredentialsProvider : public TGrpcIamCredentialsProvider(params, [token = params.OAuthToken](TRequest& req) { req.set_yandex_passport_oauth_token(TStringType{token}); - }, &TService::Stub::AsyncCreate) {} + }, &TService::Stub::async_interface::Create) {} }; template diff --git a/src/library/grpc/common/constants.h b/include/ydb-cpp-sdk/library/grpc_common/constants.h similarity index 69% rename from src/library/grpc/common/constants.h rename to include/ydb-cpp-sdk/library/grpc_common/constants.h index b67a892208..60712d4f00 100644 --- a/src/library/grpc/common/constants.h +++ b/include/ydb-cpp-sdk/library/grpc_common/constants.h @@ -2,10 +2,8 @@ #include -namespace NYdbGrpc { -inline namespace V3 { +namespace NYdb::inline V3::NGrpc { constexpr uint64_t DEFAULT_GRPC_MESSAGE_SIZE_LIMIT = 64000000; } -} diff --git a/src/client/iam/iam.cpp b/src/client/iam/iam.cpp index b3f7c59c87..5be65d306a 100644 --- a/src/client/iam/iam.cpp +++ b/src/client/iam/iam.cpp @@ -1,6 +1,6 @@ #include -#include "common/iam.h" +#include #include #include diff --git a/src/client/iam_private/common/iam.h b/src/client/iam_private/common/iam.h index bbc09b0d34..d84c0caa3a 100644 --- a/src/client/iam_private/common/iam.h +++ b/src/client/iam_private/common/iam.h @@ -1,24 +1,32 @@ #include -#include +#include namespace NYdb::inline V3 { template - class TIamServiceCredentialsProviderFactory : public ICredentialsProviderFactory { +private: + class TCredentialsProvider : public TGrpcIamCredentialsProvider { + public: + TCredentialsProvider(const TIamServiceParams& params) + : TGrpcIamCredentialsProvider(params, + [¶ms](TRequest& req) { + req.set_service_id(params.ServiceId); + req.set_microservice_id(params.MicroserviceId); + req.set_resource_id(params.ResourceId); + req.set_resource_type(params.ResourceType); + req.set_target_service_account_id(params.TargetServiceAccountId); + }, &TService::Stub::async_interface::CreateForService) {} + }; + public: - TIamServiceCredentialsProviderFactory(const TIamServiceParams& params) : Params_(params) {} + TIamServiceCredentialsProviderFactory(const TIamServiceParams& params) + : Params_(params) + {} - TCredentialsProviderPtr CreateProvider() const final { - return std::make_shared>(Params_, - [params = Params_](TRequest& req) { - req.set_service_id(params.ServiceId); - req.set_microservice_id(params.MicroserviceId); - req.set_resource_id(params.ResourceId); - req.set_resource_type(params.ResourceType); - req.set_target_service_account_id(params.TargetServiceAccountId); - }, &TService::Stub::AsyncCreateForService); + TCredentialsProviderPtr CreateProvider() const override final { + return std::make_shared(Params_); } private: diff --git a/src/library/grpc/client/grpc_client_low.h b/src/library/grpc/client/grpc_client_low.h index 3772dcba98..f1e2156e8a 100644 --- a/src/library/grpc/client/grpc_client_low.h +++ b/src/library/grpc/client/grpc_client_low.h @@ -12,7 +12,6 @@ #include #include -#include #include #include #include diff --git a/src/library/grpc/client/grpc_common.h b/src/library/grpc/client/grpc_common.h index ee96c907ec..2a260b3b20 100644 --- a/src/library/grpc/client/grpc_common.h +++ b/src/library/grpc/client/grpc_common.h @@ -4,7 +4,7 @@ #include #include -#include +#include #include #include @@ -16,7 +16,7 @@ inline namespace V3 { struct TGRpcClientConfig { std::string Locator; // format host:port TDuration Timeout = TDuration::Max(); // request timeout - ui64 MaxMessageSize = DEFAULT_GRPC_MESSAGE_SIZE_LIMIT; // Max request and response size + ui64 MaxMessageSize = NYdb::NGrpc::DEFAULT_GRPC_MESSAGE_SIZE_LIMIT; // Max request and response size ui64 MaxInboundMessageSize = 0; // overrides MaxMessageSize for incoming requests ui64 MaxOutboundMessageSize = 0; // overrides MaxMessageSize for outgoing requests ui32 MaxInFlight = 0; @@ -37,7 +37,7 @@ struct TGRpcClientConfig { TGRpcClientConfig& operator=(TGRpcClientConfig&&) = default; TGRpcClientConfig(const std::string& locator, TDuration timeout = TDuration::Max(), - ui64 maxMessageSize = DEFAULT_GRPC_MESSAGE_SIZE_LIMIT, ui32 maxInFlight = 0, const std::string& caCert = "", const std::string& clientCert = "", + ui64 maxMessageSize = NYdb::NGrpc::DEFAULT_GRPC_MESSAGE_SIZE_LIMIT, ui32 maxInFlight = 0, const std::string& caCert = "", const std::string& clientCert = "", const std::string& clientPrivateKey = "", grpc_compression_algorithm compressionAlgorithm = GRPC_COMPRESS_NONE, bool enableSsl = false) : Locator(locator) , Timeout(timeout) From fe9356e182acc80738f3d58e74cb98028a818aa5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 27 Aug 2025 14:29:01 +0000 Subject: [PATCH 24/25] Update import generation: 20 --- .github/import_generation.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/import_generation.txt b/.github/import_generation.txt index 209e3ef4b6..aabe6ec390 100644 --- a/.github/import_generation.txt +++ b/.github/import_generation.txt @@ -1 +1 @@ -20 +21 From a1878fd18987215a12c84948efa562c4ee1a3435 Mon Sep 17 00:00:00 2001 From: Bulat Gayazov Date: Thu, 28 Aug 2025 17:43:01 +0000 Subject: [PATCH 25/25] Fixed CMakeLists and increased grpc version --- .devcontainer/Dockerfile | 72 ++++++++++++------- .github/actions/prepare_vm/action.yaml | 61 ++++++++++++---- CMakePresets.json | 3 +- cmake/FindgRPC.cmake | 66 ----------------- cmake/external_libs.cmake | 2 +- examples/CMakeLists.txt | 1 + examples/time/CMakeLists.txt | 37 ++++++++++ .../client/iam/common/CMakeLists.txt | 1 + src/client/iam/CMakeLists.txt | 3 - src/client/iam/common/CMakeLists.txt | 11 --- src/client/iam_private/CMakeLists.txt | 1 - src/client/iam_private/common/CMakeLists.txt | 2 +- tests/integration/basic_example/main.cpp | 4 +- .../unit/library/operation_id/CMakeLists.txt | 1 + 14 files changed, 139 insertions(+), 126 deletions(-) delete mode 100644 cmake/FindgRPC.cmake create mode 100644 examples/time/CMakeLists.txt delete mode 100644 src/client/iam/common/CMakeLists.txt diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index b7a2b1ef82..8c9d84d085 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,53 +1,73 @@ FROM --platform=linux/amd64 mcr.microsoft.com/devcontainers/cpp:1-ubuntu-22.04 # Install software-properties-common for add-apt-repository -RUN apt-get update && apt-get -y install software-properties-common - -# Install CMake -ENV CMAKE_VERSION=3.27.7 -RUN add-apt-repository ppa:ubuntu-toolchain-r/test && apt-get -y update && apt-get -y install make && \ - wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}.tar.gz && \ - tar -xvzf cmake-${CMAKE_VERSION}.tar.gz && cd cmake-${CMAKE_VERSION} && \ - ./bootstrap && \ - make -j$(nproc) && \ - make install +RUN apt-get -y update && apt-get -y install software-properties-common && add-apt-repository ppa:ubuntu-toolchain-r/test # Install C++ tools and libraries RUN apt-get -y update && apt-get -y install \ - git gdb ninja-build libidn11-dev ragel yasm protobuf-compiler \ - protobuf-compiler-grpc libprotobuf-dev libgrpc++-dev libgrpc-dev libgrpc++1 libgrpc10 \ + git gdb ninja-build libidn11-dev ragel yasm libc-ares-dev libre2-dev \ rapidjson-dev zlib1g-dev libxxhash-dev libzstd-dev libsnappy-dev libgtest-dev libgmock-dev \ libbz2-dev libdouble-conversion-dev libstdc++-13-dev gcc-13 g++-13 liblz4-dev libssl-dev \ && apt-get clean && rm -rf /var/lib/apt/lists/* +# Install CMake +ENV CMAKE_VERSION=3.27.7 +RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.sh \ + -q -O cmake-install.sh \ + && chmod u+x cmake-install.sh \ + && ./cmake-install.sh --skip-license --prefix=/usr/local \ + && rm cmake-install.sh + # Install LLVM +ENV LLVM_VERSION=16 RUN wget https://apt.llvm.org/llvm.sh && \ chmod u+x llvm.sh && \ - ./llvm.sh 16 + ./llvm.sh ${LLVM_VERSION} # Update alternatives to use clang-16 by default -RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-16 10000 && \ - update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-16 10000 && \ - update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-16 10000 +RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${LLVM_VERSION} 10000 && \ + update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-${LLVM_VERSION} 10000 && \ + update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-${LLVM_VERSION} 10000 # Update alternatives to use gcc-13 by default RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 10000 && \ update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 10000 -# Install libiconv -ENV LIBICONV_VERSION=1.15 -RUN wget https://ftp.gnu.org/pub/gnu/libiconv/libiconv-${LIBICONV_VERSION}.tar.gz && \ - tar -xvzf libiconv-${LIBICONV_VERSION}.tar.gz && cd libiconv-${LIBICONV_VERSION} && \ - ./configure --prefix=/usr/local && \ - make && \ - make install +# Install abseil-cpp +ENV ABSEIL_CPP_VERSION=20230802.0 +RUN wget -O abseil-cpp-${ABSEIL_CPP_VERSION}.tar.gz https://github.com/abseil/abseil-cpp/archive/refs/tags/${ABSEIL_CPP_VERSION}.tar.gz && \ + tar -xvzf abseil-cpp-${ABSEIL_CPP_VERSION}.tar.gz && cd abseil-cpp-${ABSEIL_CPP_VERSION} && \ + mkdir build && cd build && \ + cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DABSL_PROPAGATE_CXX_STD=ON .. && \ + cmake --build . --config Release --target install + +# Install protobuf +ENV PROTOBUF_VERSION=3.21.12 +RUN wget -O protobuf-${PROTOBUF_VERSION}.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v${PROTOBUF_VERSION}.tar.gz && \ + tar -xvzf protobuf-${PROTOBUF_VERSION}.tar.gz && cd protobuf-${PROTOBUF_VERSION} && \ + mkdir build && cd build && \ + cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON -Dprotobuf_ABSL_PROVIDER=package .. && \ + cmake --build . --config Release --target install + +# Install grpc +ENV GRPC_VERSION=1.41.0 +RUN wget -O grpc-${GRPC_VERSION}.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v${GRPC_VERSION}.tar.gz && \ + tar -xvzf grpc-${GRPC_VERSION}.tar.gz && cd grpc-${GRPC_VERSION} && \ + mkdir build && cd build && \ + cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 \ + -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_BUILD_CSHARP_EXT=OFF \ + -DgRPC_ZLIB_PROVIDER=package -DgRPC_CARES_PROVIDER=package -DgRPC_RE2_PROVIDER=package \ + -DgRPC_SSL_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_ABSL_PROVIDER=package \ + -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF .. && \ + cmake --build . --config Release --target install # Install base64 ENV BASE64_VERSION=0.5.2 RUN wget -O base64-${BASE64_VERSION}.tar.gz https://github.com/aklomp/base64/archive/refs/tags/v${BASE64_VERSION}.tar.gz && \ tar -xvzf base64-${BASE64_VERSION}.tar.gz && cd base64-${BASE64_VERSION} && \ mkdir build && cd build && \ - cmake -DCMAKE_BUILD_TYPE=Release .. && \ + cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. && \ cmake --build . --config Release --target install # Install brotli @@ -55,7 +75,7 @@ ENV BROTLI_VERSION=1.1.0 RUN wget -O brotli-${BROTLI_VERSION}.tar.gz https://github.com/google/brotli/archive/refs/tags/v${BROTLI_VERSION}.tar.gz && \ tar -xvzf brotli-${BROTLI_VERSION}.tar.gz && cd brotli-${BROTLI_VERSION} && \ mkdir build && cd build && \ - cmake -DCMAKE_BUILD_TYPE=Release .. && \ + cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. && \ cmake --build . --config Release --target install # Install jwt-cpp @@ -63,7 +83,7 @@ ENV JWT_CPP_VERSION=0.7.0 RUN wget -O jwt-cpp-${JWT_CPP_VERSION}.tar.gz https://github.com/Thalhammer/jwt-cpp/archive/refs/tags/v${JWT_CPP_VERSION}.tar.gz && \ tar -xvzf jwt-cpp-${JWT_CPP_VERSION}.tar.gz && cd jwt-cpp-${JWT_CPP_VERSION} && \ mkdir build && cd build && \ - cmake -DCMAKE_BUILD_TYPE=Release .. && \ + cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. && \ cmake --build . --config Release --target install # Install ccache 4.8.1 or above diff --git a/.github/actions/prepare_vm/action.yaml b/.github/actions/prepare_vm/action.yaml index 7077f4a5ee..4ddc0b3c9d 100644 --- a/.github/actions/prepare_vm/action.yaml +++ b/.github/actions/prepare_vm/action.yaml @@ -11,10 +11,9 @@ runs: run: | sudo add-apt-repository ppa:ubuntu-toolchain-r/test sudo apt-get -y update - sudo apt-get -y install git ninja-build libidn11-dev ragel yasm protobuf-compiler \ - protobuf-compiler-grpc libprotobuf-dev libgrpc++-dev libgrpc-dev libgrpc++1 libgrpc10 \ + sudo apt-get -y install git ninja-build libidn11-dev ragel yasm libc-ares-dev libre2-dev \ rapidjson-dev zlib1g-dev libxxhash-dev libzstd-dev libsnappy-dev libgtest-dev libgmock-dev \ - libbz2-dev libdouble-conversion-dev libstdc++-13-dev gcc-13 g++-13 + libbz2-dev liblz4-dev libdouble-conversion-dev libstdc++-13-dev gcc-13 g++-13 wget https://apt.llvm.org/llvm.sh chmod u+x llvm.sh sudo ./llvm.sh 16 @@ -24,32 +23,64 @@ runs: sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 10000 sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 10000 - wget https://ftp.gnu.org/pub/gnu/libiconv/libiconv-1.15.tar.gz - tar -xvzf libiconv-1.15.tar.gz - cd libiconv-1.15 - ./configure --prefix=/usr/local - make - sudo make install - cd ../ + # Install abseil-cpp + wget -O abseil-cpp-20230802.0.tar.gz https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.0.tar.gz + tar -xvzf abseil-cpp-20230802.0.tar.gz + cd abseil-cpp-20230802.0 + mkdir build && cd build + cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DABSL_PROPAGATE_CXX_STD=ON .. + sudo cmake --build . --config Release --target install + cd ../../ + + # Install protobuf + wget -O protobuf-3.21.12.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz + tar -xvzf protobuf-3.21.12.tar.gz + cd protobuf-3.21.12 + mkdir build && cd build + cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON -Dprotobuf_ABSL_PROVIDER=package .. + sudo cmake --build . --config Release --target install + cd ../../ + + # Install gRPC + wget -O grpc-1.41.0.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v1.41.0.tar.gz + tar -xvzf grpc-1.41.0.tar.gz && cd grpc-1.41.0 + mkdir build && cd build + cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 \ + -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_BUILD_CSHARP_EXT=OFF \ + -DgRPC_ZLIB_PROVIDER=package -DgRPC_CARES_PROVIDER=package -DgRPC_RE2_PROVIDER=package \ + -DgRPC_SSL_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_ABSL_PROVIDER=package \ + -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF .. + sudo cmake --build . --config Release --target install + cd ../../ + + # Install base64 wget -O base64-0.5.2.tar.gz https://github.com/aklomp/base64/archive/refs/tags/v0.5.2.tar.gz tar -xvzf base64-0.5.2.tar.gz && cd base64-0.5.2 mkdir build && cd build - cmake -DCMAKE_BUILD_TYPE=Release .. + cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. sudo cmake --build . --config Release --target install cd ../../ + + # Install brotli wget -O brotli-1.1.0.tar.gz https://github.com/google/brotli/archive/refs/tags/v1.1.0.tar.gz tar -xvzf brotli-1.1.0.tar.gz && cd brotli-1.1.0 mkdir build && cd build - cmake -DCMAKE_BUILD_TYPE=Release .. + cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. sudo cmake --build . --config Release --target install cd ../../ + + # Install jwt-cpp wget -O jwt-cpp-0.7.0.tar.gz https://github.com/Thalhammer/jwt-cpp/archive/refs/tags/v0.7.0.tar.gz tar -xvzf jwt-cpp-0.7.0.tar.gz && cd jwt-cpp-0.7.0 mkdir build && cd build - cmake -DCMAKE_BUILD_TYPE=Release .. + cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. sudo cmake --build . --config Release --target install cd ../../ + + # Install ccache (V=4.8.1; curl -L https://github.com/ccache/ccache/releases/download/v${V}/ccache-${V}-linux-x86_64.tar.xz | \ sudo tar -xJ -C /usr/local/bin/ --strip-components=1 --no-same-owner ccache-${V}-linux-x86_64/ccache) - sudo rm -rf llvm.sh libiconv-1.15.tar.gz base64-0.5.2.tar.gz brotli-1.1.0.tar.gz jwt-cpp-0.7.0.tar.gz \ - libiconv-1.15 base64-0.5.2 brotli-1.1.0 jwt-cpp-0.7.0 + sudo rm -rf llvm.sh abseil-cpp-20230802.0.tar.gz protobuf-3.21.12.tar.gz grpc-1.41.0.tar.gz \ + base64-0.5.2.tar.gz brotli-1.1.0.tar.gz jwt-cpp-0.7.0.tar.gz abseil-cpp-20230802.0 \ + protobuf-3.21.12 grpc-1.41.0 base64-0.5.2 brotli-1.1.0 jwt-cpp-0.7.0 diff --git a/CMakePresets.json b/CMakePresets.json index ad610dd6dd..4432dcb354 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -10,7 +10,8 @@ "name": "base", "displayName": "Generic Config", "cacheVariables": { - "CMAKE_EXPORT_COMPILE_COMMANDS": "ON" + "CMAKE_EXPORT_COMPILE_COMMANDS": "ON", + "CMAKE_PREFIX_PATH": "~/install" }, "generator": "Ninja", "binaryDir": "${sourceDir}/build" diff --git a/cmake/FindgRPC.cmake b/cmake/FindgRPC.cmake deleted file mode 100644 index e68693b436..0000000000 --- a/cmake/FindgRPC.cmake +++ /dev/null @@ -1,66 +0,0 @@ -if (NOT Threads_FOUND) - find_package(Threads REQUIRED) -endif() - -if (NOT Protobuf_FOUND) - find_package(Protobuf REQUIRED) -endif() - -find_library(gRPC_grpc_LIBRARY - grpc - HINTS $ENV{gRPC_ROOT}/lib -) - -find_library(gRPC_grpc++_LIBRARY - grpc++ - HINTS $ENV{gRPC_ROOT}/lib -) - -find_path(gRPC_INCLUDE_DIR - grpcpp/grpcpp.h - HINTS $ENV{gRPC_ROOT}/include -) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(gRPC DEFAULT_MSG gRPC_grpc++_LIBRARY gRPC_grpc_LIBRARY gRPC_INCLUDE_DIR) - -mark_as_advanced(gRPC_grcp++_LIBRARIES gRPC_grpc_LIBRARIES gRPC_INCLUDE_DIR) - -if (gRPC_FOUND) - if (NOT TARGET gRPC::grpc) - add_library(gRPC::grpc UNKNOWN IMPORTED) - set_target_properties(gRPC::grpc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${gRPC_INCLUDE_DIR}") - set_target_properties(gRPC::grpc PROPERTIES IMPORTED_LOCATION "${gRPC_grpc_LIBRARY}") - set_property(TARGET gRPC::grpc APPEND PROPERTY INTERFACE_LINK_LIBRARIES - protobuf::libprotobuf - Threads::Threads - ) - endif() - - if (NOT TARGET gRPC::grpc++) - add_library(gRPC::grpc++ UNKNOWN IMPORTED) - set_target_properties(gRPC::grpc++ PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${gRPC_INCLUDE_DIR}") - set_target_properties(gRPC::grpc++ PROPERTIES IMPORTED_LOCATION "${gRPC_grpc++_LIBRARY}") - set_property(TARGET gRPC::grpc++ APPEND PROPERTY INTERFACE_LINK_LIBRARIES - gRPC::grpc - protobuf::libprotobuf - Threads::Threads - ) - endif() - - if (NOT TARGET gRPC::grpc_cpp_plugin) - add_executable(gRPC::grpc_cpp_plugin IMPORTED) - endif() - - get_target_property(_gRPC_CPP_PLUGIN_EXECUTABLE gRPC::grpc_cpp_plugin IMPORTED_LOCATION) - - if (NOT _gRPC_CPP_PLUGIN_EXECUTABLE) - find_program(_gRPC_CPP_PLUGIN_EXECUTABLE grpc_cpp_plugin DOC "The gRPC C++ plugin for protoc") - mark_as_advanced(_gRPC_CPP_PLUGIN_EXECUTABLE) - if (_gRPC_CPP_PLUGIN_EXECUTABLE) - set_property(TARGET gRPC::grpc_cpp_plugin PROPERTY IMPORTED_LOCATION ${_gRPC_CPP_PLUGIN_EXECUTABLE}) - else() - set(gRPC_FOUND "grpc_cpp_plugin-NOTFOUND") - endif() - endif() -endif() diff --git a/cmake/external_libs.cmake b/cmake/external_libs.cmake index 22d0603e77..dc46fdb1d5 100644 --- a/cmake/external_libs.cmake +++ b/cmake/external_libs.cmake @@ -2,7 +2,7 @@ find_package(IDN REQUIRED) find_package(Iconv REQUIRED) find_package(OpenSSL REQUIRED) find_package(Protobuf REQUIRED) -find_package(gRPC REQUIRED) +find_package(gRPC 1.41.0 REQUIRED) find_package(ZLIB REQUIRED) find_package(xxHash REQUIRED) find_package(ZSTD REQUIRED) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 118dde30ce..9607701b4f 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -3,6 +3,7 @@ add_subdirectory(bulk_upsert_simple) add_subdirectory(pagination) add_subdirectory(secondary_index) add_subdirectory(secondary_index_builtin) +add_subdirectory(time) add_subdirectory(topic_reader) add_subdirectory(topic_writer/transaction) add_subdirectory(ttl) diff --git a/examples/time/CMakeLists.txt b/examples/time/CMakeLists.txt new file mode 100644 index 0000000000..3019fd0485 --- /dev/null +++ b/examples/time/CMakeLists.txt @@ -0,0 +1,37 @@ +add_executable(time_example) + +target_link_libraries(time_example + PUBLIC + yutil + getopt + YDB-CPP-SDK::Query + YDB-CPP-SDK::Helpers +) + +target_sources(time_example + PRIVATE + main.cpp +) + +vcs_info(time_example) + +if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64") + target_link_libraries(time_example PUBLIC + cpuid_check + ) +endif() + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_link_options(time_example PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -lpthread + ) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") + target_link_options(time_example PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -framework + CoreFoundation + ) +endif() diff --git a/include/ydb-cpp-sdk/client/iam/common/CMakeLists.txt b/include/ydb-cpp-sdk/client/iam/common/CMakeLists.txt index 667ae4bd63..a3aa3438ef 100644 --- a/include/ydb-cpp-sdk/client/iam/common/CMakeLists.txt +++ b/include/ydb-cpp-sdk/client/iam/common/CMakeLists.txt @@ -4,6 +4,7 @@ target_link_libraries(client-iam-types INTERFACE client-ydb_types-credentials library-jwt + gRPC::grpc++ yutil ) diff --git a/src/client/iam/CMakeLists.txt b/src/client/iam/CMakeLists.txt index 4dd6d70785..48f2d705ad 100644 --- a/src/client/iam/CMakeLists.txt +++ b/src/client/iam/CMakeLists.txt @@ -1,5 +1,3 @@ -add_subdirectory(common) - _ydb_sdk_add_library(client-iam) target_link_libraries(client-iam @@ -8,7 +6,6 @@ target_link_libraries(client-iam yutil PRIVATE api-client-yc_public - client-iam-common json http-simple ) diff --git a/src/client/iam/common/CMakeLists.txt b/src/client/iam/common/CMakeLists.txt deleted file mode 100644 index 2d50e166c4..0000000000 --- a/src/client/iam/common/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -_ydb_sdk_add_library(client-iam-common INTERFACE) - -target_link_libraries(client-iam-common - INTERFACE - client-iam-types - grpc-client - threading-future - yutil -) - -_ydb_sdk_install_targets(client-iam-common) diff --git a/src/client/iam_private/CMakeLists.txt b/src/client/iam_private/CMakeLists.txt index 0f377d7d4e..e8e931c779 100644 --- a/src/client/iam_private/CMakeLists.txt +++ b/src/client/iam_private/CMakeLists.txt @@ -8,7 +8,6 @@ target_link_libraries(client-iam_private yutil PRIVATE api-client-yc_private - client-iam_private-common ) target_sources(client-iam_private diff --git a/src/client/iam_private/common/CMakeLists.txt b/src/client/iam_private/common/CMakeLists.txt index 244879c537..e9cb00bbc2 100644 --- a/src/client/iam_private/common/CMakeLists.txt +++ b/src/client/iam_private/common/CMakeLists.txt @@ -2,7 +2,7 @@ _ydb_sdk_add_library(client-iam_private-common INTERFACE) target_link_libraries(client-iam_private-common INTERFACE - client-iam-common + client-iam-types client-iam_private-types ) diff --git a/tests/integration/basic_example/main.cpp b/tests/integration/basic_example/main.cpp index 72c4e07dfc..082aacf23e 100644 --- a/tests/integration/basic_example/main.cpp +++ b/tests/integration/basic_example/main.cpp @@ -12,7 +12,9 @@ static void ValidateResultSet(const std::vector& columns, const std::vector>& values, const NYdb::TResultSet& rs) { Ydb::ResultSet protoResultSet; - protoResultSet.set_format(Ydb::ResultSet::FORMAT_VALUE); + if (auto version = std::getenv("YDB_VERSION"); !version || std::string(version) == "trunk") { + protoResultSet.set_format(Ydb::ResultSet::FORMAT_VALUE); + } for (const auto& column : columns) { auto* protoColumn = protoResultSet.add_columns(); diff --git a/tests/unit/library/operation_id/CMakeLists.txt b/tests/unit/library/operation_id/CMakeLists.txt index 86d3fd5131..47b9c2d5c4 100644 --- a/tests/unit/library/operation_id/CMakeLists.txt +++ b/tests/unit/library/operation_id/CMakeLists.txt @@ -4,6 +4,7 @@ add_ydb_test(NAME operation_id_ut LINK_LIBRARIES yutil cpp-testing-unittest_main + lib-operation_id-protos library-operation_id cpp-testing-unittest LABELS