From cc8a2dedff0507dbe1dfd1ea83c7e9bb5328158b Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Wed, 12 Feb 2025 23:42:49 +0000 Subject: [PATCH 01/50] WIP --- ydb/core/grpc_services/rpc_bsconfig.cpp | 14 ++--- ydb/core/grpc_services/rpc_bsconfig_base.h | 4 +- .../yaml_config/yaml_config_parser.cpp | 2 +- ydb/public/api/protos/ydb_bsconfig.proto | 28 +++++++--- .../ydb_cli/commands/ydb_dynamic_config.cpp | 52 ++++++++++++++++--- .../ydb_cli/commands/ydb_storage_config.cpp | 7 ++- .../cpp/client/ydb_types/status/status.cpp | 4 ++ .../sdk/cpp/client/ydb_types/status/status.h | 1 + .../client/bsconfig/storage_config.h | 17 ++++-- .../ydb-cpp-sdk/client/types/status/status.h | 1 + .../src/client/bsconfig/storage_config.cpp | 47 ++++++++++------- .../cpp/src/client/types/status/status.cpp | 4 ++ 12 files changed, 134 insertions(+), 47 deletions(-) diff --git a/ydb/core/grpc_services/rpc_bsconfig.cpp b/ydb/core/grpc_services/rpc_bsconfig.cpp index 96690475f80d..efe6679f71c6 100644 --- a/ydb/core/grpc_services/rpc_bsconfig.cpp +++ b/ydb/core/grpc_services/rpc_bsconfig.cpp @@ -25,7 +25,7 @@ using namespace NActors; using namespace Ydb; bool CopyToConfigRequest(const Ydb::BSConfig::ReplaceStorageConfigRequest &from, NKikimrBlobStorage::TConfigRequest *to) { - to->CopyFrom(NKikimr::NYaml::BuildInitDistributedStorageCommand(from.yaml_config())); + to->CopyFrom(NKikimr::NYaml::BuildInitDistributedStorageCommand(from.main_config())); return true; } @@ -77,7 +77,7 @@ void CopyFromConfigResponse(const NKikimrBlobStorage::TConfigResponse &from, Ydb } } storageConfig.set_item_config_generation(itemConfigGeneration); - to->set_yaml_config(NYaml::ParseProtoToYaml(storageConfig)); + to->set_main_config(NYaml::ParseProtoToYaml(storageConfig)); } class TReplaceStorageConfigRequest : public TBSConfigRequestGrpcSetYAML(GetProtoRequest()->yaml_config()); + cmd->SetYAML(GetProtoRequest()->main_config()); } void FillDistconfResult(NKikimrBlobStorage::TEvNodeConfigInvokeOnRootResult& /*record*/, @@ -105,7 +105,7 @@ class TReplaceStorageConfigRequest : public TBSConfigRequestGrpcyaml_config()); + newConfig = NYaml::Parse(GetProtoRequest()->main_config()); } catch (const std::exception&) { return false; // assuming no distconf enabled in this config } @@ -119,8 +119,8 @@ class TReplaceStorageConfigRequest : public TBSConfigRequestGrpc; return std::make_unique( - opt(&T::has_yaml_config, &T::yaml_config), - opt(&T::has_storage_yaml_config, &T::storage_yaml_config), + opt(&T::has_main_config, &T::main_config), + opt(&T::has_storage_config, &T::storage_config), opt(&T::has_switch_dedicated_storage_section, &T::switch_dedicated_storage_section), request->dedicated_config_mode()); } @@ -145,7 +145,7 @@ class TFetchStorageConfigRequest : public TBSConfigRequestGrpcGet()->Record; if (record.HasClusterYaml()) { - result.set_yaml_config(ev->Get()->Record.GetClusterYaml()); + result.set_main_config(ev->Get()->Record.GetClusterYaml()); } if (record.HasStorageYaml()) { - result.set_storage_yaml_config(ev->Get()->Record.GetStorageYaml()); + result.set_storage_config(ev->Get()->Record.GetStorageYaml()); } self->ReplyWithResult(Ydb::StatusIds::SUCCESS, result, self->ActorContext()); } else { diff --git a/ydb/library/yaml_config/yaml_config_parser.cpp b/ydb/library/yaml_config/yaml_config_parser.cpp index 3cb1f1469dfe..a7eae9e05713 100644 --- a/ydb/library/yaml_config/yaml_config_parser.cpp +++ b/ydb/library/yaml_config/yaml_config_parser.cpp @@ -1480,7 +1480,7 @@ namespace NKikimr::NYaml { Ydb::BSConfig::ReplaceStorageConfigRequest BuildReplaceDistributedStorageCommand(const TString& data) { Ydb::BSConfig::ReplaceStorageConfigRequest replaceRequest; - replaceRequest.set_yaml_config(data); + replaceRequest.set_main_config(data); return replaceRequest; } diff --git a/ydb/public/api/protos/ydb_bsconfig.proto b/ydb/public/api/protos/ydb_bsconfig.proto index d1c6c462e84c..adc97d8df868 100644 --- a/ydb/public/api/protos/ydb_bsconfig.proto +++ b/ydb/public/api/protos/ydb_bsconfig.proto @@ -15,10 +15,26 @@ import "ydb/public/api/protos/ydb_operation.proto"; message ReplaceStorageConfigRequest { Ydb.Operations.OperationParams operation_params = 1; - optional string yaml_config = 2; // cluster yaml config - optional string storage_yaml_config = 3; // dedicated storage yaml config (when dual-config mode is enabled) - optional bool switch_dedicated_storage_section = 4; // if filled, can turn on or off dedicated section of YAML config - bool dedicated_config_mode = 5; // if true, then user expects system to work in dual-config mode + + // cluster yaml config + optional string main_config = 2; + + // dedicated storage yaml config (when dual-config mode is enabled) + optional string storage_config = 3; + + // if filled, can turn on or off dedicated section of YAML config + optional bool switch_dedicated_storage_section = 4; + + // if true, then user expects system to work in dual-config mode + bool dedicated_config_mode = 5; + + bool dry_run = 7; + + bool allow_unknown_fields = 8; + bool allow_absent_database = 9; + // next two together allow to mimic DynConfigService.SetConfig behavior + bool allow_incorrect_version = 10; + bool allow_incorrect_cluster = 11; } message ReplaceStorageConfigResponse { @@ -39,8 +55,8 @@ message FetchStorageConfigResponse { } message FetchStorageConfigResult { - optional string yaml_config = 1; - optional string storage_yaml_config = 2; + optional string main_config = 1; + optional string storage_config = 2; } message BootstrapClusterRequest { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp index fbab2ff9d4fa..882aec9806ee 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp @@ -1,6 +1,7 @@ #include "ydb_dynamic_config.h" #include +#include #include #include @@ -180,15 +181,50 @@ void TCommandConfigReplace::Parse(TConfig& config) { int TCommandConfigReplace::Run(TConfig& config) { std::unique_ptr driver = std::make_unique(CreateDriver(config)); - auto client = NYdb::NDynamicConfig::TDynamicConfigClient(*driver); - auto exec = [&]() { - if (Force) { - return client.SetConfig(DynamicConfig, DryRun, AllowUnknownFields).GetValueSync(); - } + auto client = NYdb::NStorageConfig::TStorageConfigClient(*driver); + + NYdb::NStorageConfig::TReplaceStorageConfigSettings settings; + + if (Force) { + settings.AllowIncorrectVersion(); + settings.AllowIncorrectCluster(); + } + + if (DryRun) { + settings.DryRun(); + } + + if (AllowUnknownFields) { + settings.AllowUnknownFields(); + } + + // TODO absent database + + auto status = client.ReplaceStorageConfig( + DynamicConfig, + {}, + settings).GetValueSync(); + + if (status.IsUnimplementedError()) { + Cerr << "Warning: Fallback to DynamicConfig API" << Endl; + + auto client = NYdb::NDynamicConfig::TDynamicConfigClient(*driver); + + status = [&]() { + if (Force) { + return client.SetConfig( + DynamicConfig, + DryRun, + AllowUnknownFields).GetValueSync(); + } + + return client.ReplaceConfig( + DynamicConfig, + DryRun, + AllowUnknownFields).GetValueSync(); + }(); + } - return client.ReplaceConfig(DynamicConfig, DryRun, AllowUnknownFields).GetValueSync(); - }; - auto status = exec(); NStatusHelpers::ThrowOnErrorOrPrintIssues(status); if (!status.GetIssues()) { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp b/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp index fb074502ac57..d69bc7ebb2aa 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp @@ -142,7 +142,12 @@ void TCommandStorageConfigReplace::Parse(TConfig& config) { int TCommandStorageConfigReplace::Run(TConfig& config) { std::unique_ptr driver = std::make_unique(CreateDriver(config)); auto client = NYdb::NStorageConfig::TStorageConfigClient(*driver); - auto status = client.ReplaceStorageConfig(ClusterYaml, StorageYaml, SwitchDedicatedStorageSection, DedicatedConfigMode).GetValueSync(); + NYdb::NStorageConfig::TReplaceStorageConfigSettings settings; + settings + .SwitchDedicatedStorageSection(SwitchDedicatedStorageSection) + .DedicatedConfigMode(DedicatedConfigMode); + + auto status = client.ReplaceStorageConfig(ClusterYaml, StorageYaml, settings).GetValueSync(); NStatusHelpers::ThrowOnError(status); if (!status.GetIssues()) { diff --git a/ydb/public/sdk/cpp/client/ydb_types/status/status.cpp b/ydb/public/sdk/cpp/client/ydb_types/status/status.cpp index b06da2e902a3..4c58cde0b023 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/status/status.cpp +++ b/ydb/public/sdk/cpp/client/ydb_types/status/status.cpp @@ -56,6 +56,10 @@ void TStatus::CheckStatusOk(const TStringType& str) const { Impl_->CheckStatusOk(str); } +bool TStatus::IsUnimplementedError() const { + return Impl_->Status.Status == EStatus::CLIENT_CALL_UNIMPLEMENTED; +} + void TStatus::RaiseError(const TStringType& str) const { Impl_->RaiseError(str); } diff --git a/ydb/public/sdk/cpp/client/ydb_types/status/status.h b/ydb/public/sdk/cpp/client/ydb_types/status/status.h index 072c4067c935..f1a9333e79ac 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/status/status.h +++ b/ydb/public/sdk/cpp/client/ydb_types/status/status.h @@ -26,6 +26,7 @@ class TStatus { const NYql::TIssues& GetIssues() const; bool IsSuccess() const; bool IsTransportError() const; + bool IsUnimplementedError() const; const TStringType& GetEndpoint() const; const std::multimap& GetResponseMetadata() const; float GetConsumedRu() const; diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/bsconfig/storage_config.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/bsconfig/storage_config.h index a6196feded41..54e924c22fae 100644 --- a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/bsconfig/storage_config.h +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/bsconfig/storage_config.h @@ -36,8 +36,18 @@ struct TFetchStorageConfigResult : public TStatus { using TAsyncFetchStorageConfigResult = NThreading::TFuture; -struct TReplaceStorageConfigSettings : public NYdb::TOperationRequestSettings {}; +struct TReplaceStorageConfigSettings : public NYdb::TOperationRequestSettings { + FLUENT_SETTING_OPTIONAL(bool, SwitchDedicatedStorageSection); + FLUENT_SETTING_FLAG(DedicatedConfigMode); + FLUENT_SETTING_FLAG(DryRun); + FLUENT_SETTING_FLAG(AllowUnknownFields); + FLUENT_SETTING_FLAG(AllowAbsentDatabase); + FLUENT_SETTING_FLAG(AllowIncorrectVersion); + FLUENT_SETTING_FLAG(AllowIncorrectCluster); +}; + struct TFetchStorageConfigSettings : public NYdb::TOperationRequestSettings {}; + struct TBootstrapClusterSettings : public NYdb::TOperationRequestSettings {}; class TStorageConfigClient { @@ -47,10 +57,9 @@ class TStorageConfigClient { ~TStorageConfigClient(); // Replace config - TAsyncStatus ReplaceStorageConfig(const std::optional& yaml_config, + TAsyncStatus ReplaceStorageConfig( + const std::optional& yaml_config, const std::optional& storage_yaml_config, - std::optional switch_dedicated_storage_section, - bool dedicated_config_mode, const TReplaceStorageConfigSettings& settings = {}); // Fetch current cluster storage config diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/status/status.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/status/status.h index 96314b6e2d47..76d2d70934e9 100644 --- a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/status/status.h +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/status/status.h @@ -25,6 +25,7 @@ class TStatus { const NYdb::NIssue::TIssues& GetIssues() const; bool IsSuccess() const; bool IsTransportError() const; + bool IsUnimplementedError() const; const std::string& GetEndpoint() const; const std::multimap& GetResponseMetadata() const; float GetConsumedRu() const; diff --git a/ydb/public/sdk/cpp/src/client/bsconfig/storage_config.cpp b/ydb/public/sdk/cpp/src/client/bsconfig/storage_config.cpp index d8db1f0bc711..4207b9c30149 100644 --- a/ydb/public/sdk/cpp/src/client/bsconfig/storage_config.cpp +++ b/ydb/public/sdk/cpp/src/client/bsconfig/storage_config.cpp @@ -14,23 +14,31 @@ class TStorageConfigClient::TImpl : public TClientImplCommon& yaml_config, + TAsyncStatus ReplaceStorageConfig( + const std::optional& yaml_config, const std::optional& storage_yaml_config, - std::optional switch_dedicated_storage_section, - bool dedicated_config_mode, - const TReplaceStorageConfigSettings& settings) { + const TReplaceStorageConfigSettings& settings) + { auto request = MakeRequest(); if (yaml_config) { - request.set_yaml_config(*yaml_config); + request.set_main_config(*yaml_config); } + if (storage_yaml_config) { - request.set_storage_yaml_config(*storage_yaml_config); + request.set_storage_config(*storage_yaml_config); } - if (switch_dedicated_storage_section) { - request.set_switch_dedicated_storage_section(*switch_dedicated_storage_section); + + if (settings.SwitchDedicatedStorageSection_) { + request.set_switch_dedicated_storage_section(*settings.SwitchDedicatedStorageSection_); } - request.set_dedicated_config_mode(dedicated_config_mode); + + request.set_dedicated_config_mode(settings.DedicatedConfigMode_); + request.set_dry_run(settings.DryRun_); + request.set_allow_unknown_fields(settings.AllowUnknownFields_); + request.set_allow_absent_database(settings.AllowAbsentDatabase_); + request.set_allow_incorrect_version(settings.AllowIncorrectVersion_); + request.set_allow_incorrect_cluster(settings.AllowIncorrectCluster_); return RunSimple( std::move(request), @@ -39,7 +47,8 @@ class TStorageConfigClient::TImpl : public TClientImplCommon(settings); if (dedicated_storage_section) { request.set_dedicated_storage_section(true); @@ -53,8 +62,8 @@ class TStorageConfigClient::TImpl : public TClientImplCommonUnpackTo(&result)) { - config = result.yaml_config(); - storage_config = result.storage_yaml_config(); + config = result.main_config(); + storage_config = result.storage_config(); } TFetchStorageConfigResult val(TStatus(std::move(status)), std::string{std::move(config)}, @@ -88,15 +97,17 @@ TStorageConfigClient::TStorageConfigClient(const TDriver& driver, const TCommonC TStorageConfigClient::~TStorageConfigClient() = default; -TAsyncStatus TStorageConfigClient::ReplaceStorageConfig(const std::optional& yaml_config, - const std::optional& storage_yaml_config, std::optional switch_dedicated_storage_section, - bool dedicated_config_mode, const TReplaceStorageConfigSettings& settings) { - return Impl_->ReplaceStorageConfig(yaml_config, storage_yaml_config, switch_dedicated_storage_section, - dedicated_config_mode, settings); +TAsyncStatus TStorageConfigClient::ReplaceStorageConfig( + const std::optional& yaml_config, + const std::optional& storage_yaml_config, + const TReplaceStorageConfigSettings& settings) +{ + return Impl_->ReplaceStorageConfig(yaml_config, storage_yaml_config, settings); } TAsyncFetchStorageConfigResult TStorageConfigClient::FetchStorageConfig(bool dedicated_storage_section, - bool dedicated_cluster_section, const TFetchStorageConfigSettings& settings) { + bool dedicated_cluster_section, const TFetchStorageConfigSettings& settings) +{ return Impl_->FetchStorageConfig(dedicated_storage_section, dedicated_cluster_section, settings); } diff --git a/ydb/public/sdk/cpp/src/client/types/status/status.cpp b/ydb/public/sdk/cpp/src/client/types/status/status.cpp index 1764499e40dc..be7e44082a78 100644 --- a/ydb/public/sdk/cpp/src/client/types/status/status.cpp +++ b/ydb/public/sdk/cpp/src/client/types/status/status.cpp @@ -54,6 +54,10 @@ bool TStatus::IsTransportError() const { && static_cast(Impl_->Status.Status) <= TRANSPORT_STATUSES_LAST; } +bool TStatus::IsUnimplementedError() const { + return Impl_->Status.Status == EStatus::CLIENT_CALL_UNIMPLEMENTED; +} + void TStatus::CheckStatusOk(const std::string& str) const { Impl_->CheckStatusOk(str); } From 4635e89f5c3138a0e36ae3657418b70419b207b2 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Thu, 13 Feb 2025 08:38:37 +0000 Subject: [PATCH 02/50] WIP --- .../base/blobstorage_console_events.h | 24 +++++++++-- .../nodewarden/distconf_console.cpp | 3 +- .../cms/console/console_configs_manager.cpp | 2 + ydb/core/cms/console/console_handshake.cpp | 40 ++++++++++++++++--- ydb/core/grpc_services/rpc_bsconfig.cpp | 5 ++- .../mind/bscontroller/console_interaction.cpp | 22 +++++++--- .../mind/bscontroller/console_interaction.h | 2 + ydb/core/protos/blobstorage.proto | 10 +++++ .../yaml_config/yaml_config_parser.cpp | 13 ++++-- 9 files changed, 101 insertions(+), 20 deletions(-) diff --git a/ydb/core/blobstorage/base/blobstorage_console_events.h b/ydb/core/blobstorage/base/blobstorage_console_events.h index 37efc732f500..f6e27f97a078 100644 --- a/ydb/core/blobstorage/base/blobstorage_console_events.h +++ b/ydb/core/blobstorage/base/blobstorage_console_events.h @@ -32,8 +32,16 @@ namespace NKikimr { NKikimrBlobStorage::TEvControllerConsoleCommitRequest, EvControllerConsoleCommitRequest> { TEvControllerConsoleCommitRequest() = default; - TEvControllerConsoleCommitRequest(const TString& yamlConfig) { + TEvControllerConsoleCommitRequest( + const TString& yamlConfig, + bool allowUnknownFields = false, + bool allowIncorrectVersion = false, + bool allowIncorrectCluster = false) { + Record.SetYAML(yamlConfig); + Record.SetAllowUnknownFields(allowUnknownFields); + Record.SetAllowIncorrectVersion(allowIncorrectVersion); + Record.SetAllowIncorrectCluster(allowIncorrectCluster); } TString ToString() const override { @@ -76,8 +84,15 @@ namespace NKikimr { NKikimrBlobStorage::TEvControllerReplaceConfigRequest, EvControllerReplaceConfigRequest> { TEvControllerReplaceConfigRequest() = default; - TEvControllerReplaceConfigRequest(std::optional clusterYaml, std::optional storageYaml, - std::optional switchDedicatedStorageSection, bool dedicatedConfigMode) { + TEvControllerReplaceConfigRequest( + std::optional clusterYaml, + std::optional storageYaml, + std::optional switchDedicatedStorageSection, + bool dedicatedConfigMode, + bool allowUnknownFields, + bool allowIncorrectVersion, + bool allowIncorrectCluster) { + if (clusterYaml) { Record.SetClusterYaml(*clusterYaml); } @@ -88,6 +103,9 @@ namespace NKikimr { Record.SetSwitchDedicatedStorageSection(*switchDedicatedStorageSection); } Record.SetDedicatedConfigMode(dedicatedConfigMode); + Record.SetAllowUnknownFields(allowUnknownFields); + Record.SetAllowIncorrectVersion(allowIncorrectVersion); + Record.SetAllowIncorrectCluster(allowIncorrectCluster); } TString ToString() const override { diff --git a/ydb/core/blobstorage/nodewarden/distconf_console.cpp b/ydb/core/blobstorage/nodewarden/distconf_console.cpp index 90fd9fac9496..50d5102d481a 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_console.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_console.cpp @@ -103,7 +103,8 @@ namespace NKikimr::NStorage { } NTabletPipe::SendData(SelfId(), ConsolePipeId, new TEvBlobStorage::TEvControllerConsoleCommitRequest( - StorageConfigYaml), ++CommitRequestCookie); + StorageConfigYaml), // FIXME + ++CommitRequestCookie); break; } diff --git a/ydb/core/cms/console/console_configs_manager.cpp b/ydb/core/cms/console/console_configs_manager.cpp index b47b41751186..8df5773a83c8 100644 --- a/ydb/core/cms/console/console_configs_manager.cpp +++ b/ydb/core/cms/console/console_configs_manager.cpp @@ -113,6 +113,8 @@ void TConfigsManager::ValidateMainConfig(TUpdateConfigOpContext& opCtx) { } } catch (const yexception &e) { opCtx.Error = e.what(); + } catch (const std::exception& e) { + opCtx.Error = e.what(); } } diff --git a/ydb/core/cms/console/console_handshake.cpp b/ydb/core/cms/console/console_handshake.cpp index b2ec3f533b8e..b0c96004617b 100644 --- a/ydb/core/cms/console/console_handshake.cpp +++ b/ydb/core/cms/console/console_handshake.cpp @@ -16,9 +16,19 @@ namespace NKikimr::NConsole { class TConfigsManager::TConsoleCommitActor : public TActorBootstrapped { public: - TConsoleCommitActor(TActorId senderId, const TString& mainYamlConfig, TActorId interconnectSession, ui64 cookie) + TConsoleCommitActor( + TActorId senderId, + const TString& mainYamlConfig, + bool allowUnknownFields, + bool allowIncorrectVersion, + bool allowIncorrectCluster, + TActorId interconnectSession, + ui64 cookie) : SenderId(senderId) , MainYamlConfig(mainYamlConfig) + , AllowUnknownFields(allowUnknownFields) + , AllowIncorrectVersion(allowIncorrectVersion) + , AllowIncorrectCluster(allowIncorrectCluster) , InterconnectSession(interconnectSession) , Cookie(cookie) {} @@ -26,6 +36,9 @@ class TConfigsManager::TConsoleCommitActor : public TActorBootstrapped(); request->Record.MutableRequest()->set_config(MainYamlConfig); + request->Record.MutableRequest()->set_allow_unknown_fields(AllowUnknownFields); + // FIXME: handle force + Y_UNUSED(AllowIncorrectVersion, AllowIncorrectCluster); Send(consoleId, request.release()); Become(&TThis::StateWork); @@ -55,6 +68,9 @@ class TConfigsManager::TConsoleCommitActor : public TActorBootstrapped(); - const auto& mainYamlConfig = ev->Get()->Record.GetYAML(); + auto& record = ev->Get()->Record; + const auto& mainYamlConfig = record.GetYAML(); + bool allowUnknownFields = record.GetAllowUnknownFields(); + bool allowIncorrectVersion = record.GetAllowIncorrectVersion(); + bool allowIncorrectCluster = record.GetAllowIncorrectCluster(); if (!CheckSession(*ev, response, NKikimrBlobStorage::TEvControllerConsoleCommitResponse::SessionMismatch)) { return; } - IActor* actor = new TConsoleCommitActor(ev->Sender, mainYamlConfig, ev->InterconnectSession, ev->Cookie); + IActor* actor = new TConsoleCommitActor( + ev->Sender, + mainYamlConfig, + allowUnknownFields, + allowIncorrectVersion, + allowIncorrectCluster, + ev->InterconnectSession, + ev->Cookie); CommitActor = Register(actor); } @@ -138,17 +165,18 @@ void TConfigsManager::Handle(TEvBlobStorage::TEvControllerValidateConfigRequest: TUpdateConfigOpContext opCtx; ReplaceMainConfigMetadata(mainYamlConfig, false, opCtx); ValidateMainConfig(opCtx); + bool hasForbiddenUnknownFields = !opCtx.UnknownFields.empty() && !ev->Get()->Record.GetAllowUnknownFields(); - if (opCtx.Error || !opCtx.UnknownFields.empty()) { + if (opCtx.Error || hasForbiddenUnknownFields) { record.SetStatus(NKikimrBlobStorage::TEvControllerValidateConfigResponse::ConfigNotValid); TStringStream s; if (opCtx.Error) { s << *opCtx.Error << (opCtx.UnknownFields.empty() ? "" : " and "); } - if (!opCtx.UnknownFields.empty()) { + if (hasForbiddenUnknownFields) { s << "has forbidden unknown fields"; } - record.SetErrorReason(s.Str()); + record.SetErrorReason(s.Str()); // TODO get warnings back } else { record.SetStatus(NKikimrBlobStorage::TEvControllerValidateConfigResponse::ConfigIsValid); record.SetYAML(opCtx.UpdatedConfig); diff --git a/ydb/core/grpc_services/rpc_bsconfig.cpp b/ydb/core/grpc_services/rpc_bsconfig.cpp index efe6679f71c6..39b25dc70aa6 100644 --- a/ydb/core/grpc_services/rpc_bsconfig.cpp +++ b/ydb/core/grpc_services/rpc_bsconfig.cpp @@ -122,7 +122,10 @@ class TReplaceStorageConfigRequest : public TBSConfigRequestGrpcdedicated_config_mode()); + request->dedicated_config_mode(), + request->allow_unknown_fields(), + request->allow_incorrect_version(), + request->allow_incorrect_cluster()); } }; diff --git a/ydb/core/mind/bscontroller/console_interaction.cpp b/ydb/core/mind/bscontroller/console_interaction.cpp index caff978e2d8e..e67fd7b3f096 100644 --- a/ydb/core/mind/bscontroller/console_interaction.cpp +++ b/ydb/core/mind/bscontroller/console_interaction.cpp @@ -89,7 +89,11 @@ namespace NKikimr::NBsController { if (ConsolePipe) { Y_ABORT_UNLESS(Self.YamlConfig); if (const auto& [yaml, configVersion, yamlReturnedByFetch] = *Self.YamlConfig; yaml) { - NTabletPipe::SendData(Self.SelfId(), ConsolePipe, new TEvBlobStorage::TEvControllerConsoleCommitRequest(yaml)); + NTabletPipe::SendData( + Self.SelfId(), + ConsolePipe, + new TEvBlobStorage::TEvControllerConsoleCommitRequest( + yaml, AllowUnknownFields)); // FIXME } } break; @@ -145,7 +149,11 @@ namespace NKikimr::NBsController { if (ConsolePipe) { Y_ABORT_UNLESS(Self.YamlConfig); if (const auto& [yaml, configVersion, yamlReturnedByFetch] = *Self.YamlConfig; yaml) { - NTabletPipe::SendData(Self.SelfId(), ConsolePipe, new TEvBlobStorage::TEvControllerConsoleCommitRequest(yaml)); + NTabletPipe::SendData( + Self.SelfId(), + ConsolePipe, + new TEvBlobStorage::TEvControllerConsoleCommitRequest( + yaml, AllowUnknownFields)); // FIXME } } else { Y_ABORT_UNLESS(!ClientId); @@ -173,7 +181,7 @@ namespace NKikimr::NBsController { case NKikimrBlobStorage::TEvControllerConsoleCommitResponse::NotCommitted: STLOG(PRI_CRIT, BS_CONTROLLER, BSC28, "Console config not committed"); - Y_DEBUG_ABORT(); + Y_DEBUG_ABORT_S(record.GetErrorReason()); break; case NKikimrBlobStorage::TEvControllerConsoleCommitResponse::Committed: @@ -257,6 +265,7 @@ namespace NKikimr::NBsController { if (record.HasClusterYaml()) { PendingYamlConfig.emplace(record.GetClusterYaml()); + AllowUnknownFields = record.GetAllowUnknownFields(); } else { PendingYamlConfig.reset(); } @@ -288,6 +297,9 @@ namespace NKikimr::NBsController { auto validateConfigEv = std::make_unique(); validateConfigEv->Record.SetYAML(record.GetClusterYaml()); + validateConfigEv->Record.SetAllowUnknownFields(record.GetAllowUnknownFields()); + validateConfigEv->Record.SetAllowIncorrectVersion(record.GetAllowIncorrectVersion()); + validateConfigEv->Record.SetAllowIncorrectCluster(record.GetAllowIncorrectCluster()); NTabletPipe::SendData(Self.SelfId(), ConsolePipe, validateConfigEv.release()); } @@ -358,14 +370,14 @@ namespace NKikimr::NBsController { auto parseConfig = [&](const TString& yaml, NKikimrConfig::TAppConfig& appConfig, ui64& version) { try { auto json = NYaml::Yaml2Json(YAML::Load(yaml), true); - NYaml::Parse(json, NYaml::GetJsonToProtoConfig(), appConfig, true); + NYaml::Parse(json, NYaml::GetJsonToProtoConfig(AllowUnknownFields), appConfig, true); if (json.Has("metadata")) { if (auto& metadata = json["metadata"]; metadata.Has("version")) { version = metadata["version"].GetUIntegerRobust(); } } } catch (const std::exception& ex) { - throw TExError(TStringBuilder() << "failed to parse YAML config: " << ex.what()); + throw TExError(TStringBuilder() << "failed to parse YAML config: " << ex.what() << "\n" << yaml); } }; diff --git a/ydb/core/mind/bscontroller/console_interaction.h b/ydb/core/mind/bscontroller/console_interaction.h index 5a9d9f834caf..40d356c3a8d6 100644 --- a/ydb/core/mind/bscontroller/console_interaction.h +++ b/ydb/core/mind/bscontroller/console_interaction.h @@ -47,6 +47,8 @@ namespace NKikimr::NBsController { bool CommitInProgress = false; std::optional PendingYamlConfig; + bool AllowUnknownFields = false; + std::optional> PendingStorageYamlConfig; void MakeCommitToConsole(TString& config, ui32 configVersion); diff --git a/ydb/core/protos/blobstorage.proto b/ydb/core/protos/blobstorage.proto index 89ad31aa17ce..a0d07d1f79b4 100644 --- a/ydb/core/protos/blobstorage.proto +++ b/ydb/core/protos/blobstorage.proto @@ -1415,6 +1415,9 @@ message TEvControllerProposeConfigResponse { message TEvControllerConsoleCommitRequest { optional string YAML = 1; + optional bool AllowUnknownFields = 2; + optional bool AllowIncorrectVersion = 3; + optional bool AllowIncorrectCluster = 4; } message TEvControllerConsoleCommitResponse { @@ -1435,6 +1438,10 @@ message TEvControllerReplaceConfigRequest { optional bool SkipConsoleValidation = 5; optional bool SwitchDedicatedStorageSection = 6; optional bool DedicatedConfigMode = 7; + // console flags + optional bool AllowUnknownFields = 8; + optional bool AllowIncorrectVersion = 9; + optional bool AllowIncorrectCluster = 10; } message TEvControllerReplaceConfigResponse { @@ -1454,6 +1461,9 @@ message TEvControllerReplaceConfigResponse { message TEvControllerValidateConfigRequest { optional string YAML = 1; + optional bool AllowUnknownFields = 2; + optional bool AllowIncorrectVersion = 3; + optional bool AllowIncorrectCluster = 4; } message TEvControllerValidateConfigResponse { diff --git a/ydb/library/yaml_config/yaml_config_parser.cpp b/ydb/library/yaml_config/yaml_config_parser.cpp index a7eae9e05713..1a594fc11bae 100644 --- a/ydb/library/yaml_config/yaml_config_parser.cpp +++ b/ydb/library/yaml_config/yaml_config_parser.cpp @@ -699,8 +699,10 @@ namespace NKikimr::NYaml { auto& vdiskLoc = ctx.CombinedDiskInfo[TCombinedDiskInfoKey{}]; vdiskLoc.SetNodeID("1"); - vdiskLoc.SetPath(drivePath.value()); - vdiskLoc.SetPDiskCategory(diskType.value()); + if (drivePath && diskType) { + vdiskLoc.SetPath(drivePath.value()); + vdiskLoc.SetPDiskCategory(diskType.value()); + } } if (!config.HasChannelProfileConfig()) { @@ -710,7 +712,9 @@ namespace NKikimr::NYaml { auto& channel = *channelProfile.AddChannel(); channel.SetErasureSpecies(erasureName); channel.SetPDiskCategory(1); - channel.SetStoragePoolKind(diskTypeLower.value()); + if (diskTypeLower) { + channel.SetStoragePoolKind(diskTypeLower.value()); + } }; } } @@ -1417,7 +1421,8 @@ namespace NKikimr::NYaml { void TransformProtoConfig(TTransformContext& ctx, NKikimrConfig::TAppConfig& config, NKikimrConfig::TEphemeralInputFields& ephemeralConfig, bool relaxed) { PrepareHosts(ephemeralConfig); MoveFields(ctx, config, ephemeralConfig); - ApplyDefaultConfigs(ctx, config, ephemeralConfig); + // FIXME + //ApplyDefaultConfigs(ctx, config, ephemeralConfig); PrepareNameserviceConfig(config, ephemeralConfig); PrepareStaticGroup(ctx, config, ephemeralConfig); PrepareBlobStorageConfig(config, ephemeralConfig); From a830f358511afc9e4647eed8537b02e29b135d44 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Thu, 13 Feb 2025 09:04:31 +0000 Subject: [PATCH 03/50] WIP --- ydb/core/cms/console/console_handshake.cpp | 35 +++++++++++++++---- .../mind/bscontroller/console_interaction.cpp | 14 ++++++-- .../mind/bscontroller/console_interaction.h | 2 ++ 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/ydb/core/cms/console/console_handshake.cpp b/ydb/core/cms/console/console_handshake.cpp index b0c96004617b..a8bdde7694bc 100644 --- a/ydb/core/cms/console/console_handshake.cpp +++ b/ydb/core/cms/console/console_handshake.cpp @@ -34,12 +34,27 @@ class TConfigsManager::TConsoleCommitActor : public TActorBootstrapped(); - request->Record.MutableRequest()->set_config(MainYamlConfig); - request->Record.MutableRequest()->set_allow_unknown_fields(AllowUnknownFields); - // FIXME: handle force - Y_UNUSED(AllowIncorrectVersion, AllowIncorrectCluster); - Send(consoleId, request.release()); + if (AllowIncorrectVersion xor AllowIncorrectCluster) { + auto response = std::make_unique(); + response->Record.SetStatus(NKikimrBlobStorage::TEvControllerConsoleCommitResponse::NotCommitted); + response->Record.SetErrorReason("Options AllowIncorrectVersion and AllowIncorrectCluster currently can be used only together"); + SendInReply(std::move(response)); + PassAway(); + } + + auto executeRequest = [&](auto& request) { + request->Record.MutableRequest()->set_config(MainYamlConfig); + request->Record.MutableRequest()->set_allow_unknown_fields(AllowUnknownFields); + Send(consoleId, request.release()); + }; + + if (AllowIncorrectVersion && AllowIncorrectCluster) { + auto request = std::make_unique(); + executeRequest(request); + } else { + auto request = std::make_unique(); + executeRequest(request); + } Become(&TThis::StateWork); } @@ -51,6 +66,13 @@ class TConfigsManager::TConsoleCommitActor : public TActorBootstrapped(); + response->Record.SetStatus(NKikimrBlobStorage::TEvControllerConsoleCommitResponse::Committed); + SendInReply(std::move(response)); + PassAway(); + } + void Handle(TEvConsole::TEvGenericError::TPtr& ev) { auto response = std::make_unique(); response->Record.SetStatus(NKikimrBlobStorage::TEvControllerConsoleCommitResponse::NotCommitted); @@ -61,6 +83,7 @@ class TConfigsManager::TConsoleCommitActor : public TActorBootstrapped PendingYamlConfig; bool AllowUnknownFields = false; + bool AllowIncorrectVersion = false; + bool AllowIncorrectCluster = false; std::optional> PendingStorageYamlConfig; From d3b48aabdbb2896bdaa0987480be454346b30d43 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Thu, 13 Feb 2025 09:35:48 +0000 Subject: [PATCH 04/50] WIP --- ydb/core/grpc_services/rpc_bsconfig.cpp | 28 +++++++------- ydb/core/grpc_services/rpc_bsconfig_base.h | 12 +++--- .../yaml_config/yaml_config_parser.cpp | 4 +- ydb/library/yaml_config/yaml_config_parser.h | 2 +- ydb/public/api/grpc/ydb_bsconfig_v1.proto | 4 +- ydb/public/api/protos/ydb_bsconfig.proto | 12 +++--- .../ydb_cli/commands/ydb_dynamic_config.cpp | 4 +- .../ydb_cli/commands/ydb_storage_config.cpp | 6 +-- .../client/bsconfig/storage_config.h | 18 ++++----- .../src/client/bsconfig/storage_config.cpp | 38 +++++++++---------- ydb/services/bsconfig/grpc_service.cpp | 4 +- 11 files changed, 66 insertions(+), 66 deletions(-) diff --git a/ydb/core/grpc_services/rpc_bsconfig.cpp b/ydb/core/grpc_services/rpc_bsconfig.cpp index 39b25dc70aa6..6ea35e62f40b 100644 --- a/ydb/core/grpc_services/rpc_bsconfig.cpp +++ b/ydb/core/grpc_services/rpc_bsconfig.cpp @@ -12,11 +12,11 @@ namespace NKikimr::NGRpcService { using TEvReplaceStorageConfigRequest = - TGrpcRequestOperationCall; + TGrpcRequestOperationCall; using TEvFetchStorageConfigRequest = - TGrpcRequestOperationCall; + TGrpcRequestOperationCall; using TEvBootstrapClusterRequest = TGrpcRequestOperationCall; @@ -24,21 +24,21 @@ using TEvBootstrapClusterRequest = using namespace NActors; using namespace Ydb; -bool CopyToConfigRequest(const Ydb::BSConfig::ReplaceStorageConfigRequest &from, NKikimrBlobStorage::TConfigRequest *to) { +bool CopyToConfigRequest(const Ydb::BSConfig::ReplaceConfigRequest &from, NKikimrBlobStorage::TConfigRequest *to) { to->CopyFrom(NKikimr::NYaml::BuildInitDistributedStorageCommand(from.main_config())); return true; } -void CopyFromConfigResponse(const NKikimrBlobStorage::TConfigResponse &/*from*/, Ydb::BSConfig::ReplaceStorageConfigResult* /*to*/) { +void CopyFromConfigResponse(const NKikimrBlobStorage::TConfigResponse &/*from*/, Ydb::BSConfig::ReplaceConfigResult* /*to*/) { } -bool CopyToConfigRequest(const Ydb::BSConfig::FetchStorageConfigRequest &/*from*/, NKikimrBlobStorage::TConfigRequest *to) { +bool CopyToConfigRequest(const Ydb::BSConfig::FetchConfigRequest &/*from*/, NKikimrBlobStorage::TConfigRequest *to) { to->AddCommand()->MutableReadHostConfig(); to->AddCommand()->MutableReadBox(); return true; } -void CopyFromConfigResponse(const NKikimrBlobStorage::TConfigResponse &from, Ydb::BSConfig::FetchStorageConfigResult *to) { +void CopyFromConfigResponse(const NKikimrBlobStorage::TConfigResponse &from, Ydb::BSConfig::FetchConfigResult *to) { auto hostConfigStatus = from.GetStatus()[0]; auto boxStatus = from.GetStatus()[1]; NKikimrConfig::StorageConfig storageConfig; @@ -81,9 +81,9 @@ void CopyFromConfigResponse(const NKikimrBlobStorage::TConfigResponse &from, Ydb } class TReplaceStorageConfigRequest : public TBSConfigRequestGrpc { + Ydb::BSConfig::ReplaceConfigResult> { public: - using TBase = TBSConfigRequestGrpc; + using TBase = TBSConfigRequestGrpc; using TBase::TBase; bool ValidateRequest(Ydb::StatusIds::StatusCode& /*status*/, NYql::TIssues& /*issues*/) override { @@ -99,7 +99,7 @@ class TReplaceStorageConfigRequest : public TBSConfigRequestGrpc { + Ydb::BSConfig::FetchConfigResult> { public: - using TBase = TBSConfigRequestGrpc; + using TBase = TBSConfigRequestGrpc; using TBase::TBase; bool ValidateRequest(Ydb::StatusIds::StatusCode& /*status*/, NYql::TIssues& /*issues*/) override { @@ -147,7 +147,7 @@ class TFetchStorageConfigRequest : public TBSConfigRequestGrpc class TBaseBSConfigRequest { @@ -281,7 +281,7 @@ class TBSConfigRequestGrpc : public TRpcOperationRequestActor) { + if constexpr (std::is_same_v) { TResultRecord result; const auto& record = ev->Get()->Record; if (record.HasClusterYaml()) { @@ -299,7 +299,7 @@ class TBSConfigRequestGrpc : public TRpcOperationRequestActor) { + if constexpr (std::is_same_v) { const auto& record = ev->Get()->Record; if (record.GetStatus() == NKikimrBlobStorage::TEvControllerReplaceConfigResponse::Success) { TResultRecord result; diff --git a/ydb/library/yaml_config/yaml_config_parser.cpp b/ydb/library/yaml_config/yaml_config_parser.cpp index 1a594fc11bae..ead77a38d1ec 100644 --- a/ydb/library/yaml_config/yaml_config_parser.cpp +++ b/ydb/library/yaml_config/yaml_config_parser.cpp @@ -1483,8 +1483,8 @@ namespace NKikimr::NYaml { return result; } - Ydb::BSConfig::ReplaceStorageConfigRequest BuildReplaceDistributedStorageCommand(const TString& data) { - Ydb::BSConfig::ReplaceStorageConfigRequest replaceRequest; + Ydb::BSConfig::ReplaceConfigRequest BuildReplaceDistributedStorageCommand(const TString& data) { + Ydb::BSConfig::ReplaceConfigRequest replaceRequest; replaceRequest.set_main_config(data); return replaceRequest; } diff --git a/ydb/library/yaml_config/yaml_config_parser.h b/ydb/library/yaml_config/yaml_config_parser.h index de3a0b4221b7..2b3b7419e53b 100644 --- a/ydb/library/yaml_config/yaml_config_parser.h +++ b/ydb/library/yaml_config/yaml_config_parser.h @@ -56,7 +56,7 @@ namespace NKikimr::NYaml { TSimpleSharedPtr unknownFieldsCollector = nullptr); NKikimrBlobStorage::TConfigRequest BuildInitDistributedStorageCommand(const TString& data); - Ydb::BSConfig::ReplaceStorageConfigRequest BuildReplaceDistributedStorageCommand(const TString& data); + Ydb::BSConfig::ReplaceConfigRequest BuildReplaceDistributedStorageCommand(const TString& data); TString ParseProtoToYaml(const NKikimrConfig::StorageConfig& protoConfig); void ExtractExtraFields(NJson::TJsonValue& json, TTransformContext& ctx); diff --git a/ydb/public/api/grpc/ydb_bsconfig_v1.proto b/ydb/public/api/grpc/ydb_bsconfig_v1.proto index ee880d0abd4d..ef8d915aa5b1 100644 --- a/ydb/public/api/grpc/ydb_bsconfig_v1.proto +++ b/ydb/public/api/grpc/ydb_bsconfig_v1.proto @@ -11,10 +11,10 @@ import "ydb/public/api/protos/ydb_bsconfig.proto"; service BSConfigService { // Initialize Blobstorage/single config - rpc ReplaceStorageConfig(BSConfig.ReplaceStorageConfigRequest) returns (BSConfig.ReplaceStorageConfigResponse); + rpc ReplaceConfig(BSConfig.ReplaceConfigRequest) returns (BSConfig.ReplaceConfigResponse); // Fetch Blobstorage/single config - rpc FetchStorageConfig(BSConfig.FetchStorageConfigRequest) returns (BSConfig.FetchStorageConfigResponse); + rpc FetchConfig(BSConfig.FetchConfigRequest) returns (BSConfig.FetchConfigResponse); // Bootstrap automatically configured cluster rpc BootstrapCluster(BSConfig.BootstrapClusterRequest) returns (BSConfig.BootstrapClusterResponse); diff --git a/ydb/public/api/protos/ydb_bsconfig.proto b/ydb/public/api/protos/ydb_bsconfig.proto index adc97d8df868..9888b589376a 100644 --- a/ydb/public/api/protos/ydb_bsconfig.proto +++ b/ydb/public/api/protos/ydb_bsconfig.proto @@ -13,7 +13,7 @@ import "ydb/public/api/protos/ydb_operation.proto"; // BSConfig API. // -message ReplaceStorageConfigRequest { +message ReplaceConfigRequest { Ydb.Operations.OperationParams operation_params = 1; // cluster yaml config @@ -37,24 +37,24 @@ message ReplaceStorageConfigRequest { bool allow_incorrect_cluster = 11; } -message ReplaceStorageConfigResponse { +message ReplaceConfigResponse { Ydb.Operations.Operation operation = 1; } -message ReplaceStorageConfigResult { +message ReplaceConfigResult { } -message FetchStorageConfigRequest { +message FetchConfigRequest { Ydb.Operations.OperationParams operation_params = 1; bool dedicated_storage_section = 2; bool dedicated_cluster_section = 3; } -message FetchStorageConfigResponse { +message FetchConfigResponse { Ydb.Operations.Operation operation = 1; } -message FetchStorageConfigResult { +message FetchConfigResult { optional string main_config = 1; optional string storage_config = 2; } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp index 882aec9806ee..1c13650a3e02 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp @@ -183,7 +183,7 @@ int TCommandConfigReplace::Run(TConfig& config) { std::unique_ptr driver = std::make_unique(CreateDriver(config)); auto client = NYdb::NStorageConfig::TStorageConfigClient(*driver); - NYdb::NStorageConfig::TReplaceStorageConfigSettings settings; + NYdb::NStorageConfig::TReplaceConfigSettings settings; if (Force) { settings.AllowIncorrectVersion(); @@ -200,7 +200,7 @@ int TCommandConfigReplace::Run(TConfig& config) { // TODO absent database - auto status = client.ReplaceStorageConfig( + auto status = client.ReplaceConfig( DynamicConfig, {}, settings).GetValueSync(); diff --git a/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp b/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp index d69bc7ebb2aa..057a5ed7b3c2 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp @@ -63,7 +63,7 @@ void TCommandStorageConfigFetch::Parse(TConfig& config) { int TCommandStorageConfigFetch::Run(TConfig& config) { auto driver = std::make_unique(CreateDriver(config)); auto client = NYdb::NStorageConfig::TStorageConfigClient(*driver); - auto result = client.FetchStorageConfig(DedicatedStorageSection, DedicatedClusterSection).GetValueSync(); + auto result = client.FetchConfig(DedicatedStorageSection, DedicatedClusterSection).GetValueSync(); NStatusHelpers::ThrowOnError(result); const auto& clusterConfig = result.GetConfig(); @@ -142,12 +142,12 @@ void TCommandStorageConfigReplace::Parse(TConfig& config) { int TCommandStorageConfigReplace::Run(TConfig& config) { std::unique_ptr driver = std::make_unique(CreateDriver(config)); auto client = NYdb::NStorageConfig::TStorageConfigClient(*driver); - NYdb::NStorageConfig::TReplaceStorageConfigSettings settings; + NYdb::NStorageConfig::TReplaceConfigSettings settings; settings .SwitchDedicatedStorageSection(SwitchDedicatedStorageSection) .DedicatedConfigMode(DedicatedConfigMode); - auto status = client.ReplaceStorageConfig(ClusterYaml, StorageYaml, settings).GetValueSync(); + auto status = client.ReplaceConfig(ClusterYaml, StorageYaml, settings).GetValueSync(); NStatusHelpers::ThrowOnError(status); if (!status.GetIssues()) { diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/bsconfig/storage_config.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/bsconfig/storage_config.h index 54e924c22fae..33f56c28bf72 100644 --- a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/bsconfig/storage_config.h +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/bsconfig/storage_config.h @@ -11,8 +11,8 @@ namespace NYdb::inline V3::NStorageConfig { -struct TFetchStorageConfigResult : public TStatus { - TFetchStorageConfigResult( +struct TFetchConfigResult : public TStatus { + TFetchConfigResult( TStatus&& status, std::string&& config, std::string&& storage_config) @@ -34,9 +34,9 @@ struct TFetchStorageConfigResult : public TStatus { std::string StorageConfig_; }; -using TAsyncFetchStorageConfigResult = NThreading::TFuture; +using TAsyncFetchConfigResult = NThreading::TFuture; -struct TReplaceStorageConfigSettings : public NYdb::TOperationRequestSettings { +struct TReplaceConfigSettings : public NYdb::TOperationRequestSettings { FLUENT_SETTING_OPTIONAL(bool, SwitchDedicatedStorageSection); FLUENT_SETTING_FLAG(DedicatedConfigMode); FLUENT_SETTING_FLAG(DryRun); @@ -46,7 +46,7 @@ struct TReplaceStorageConfigSettings : public NYdb::TOperationRequestSettings {}; +struct TFetchConfigSettings : public NYdb::TOperationRequestSettings {}; struct TBootstrapClusterSettings : public NYdb::TOperationRequestSettings {}; @@ -57,14 +57,14 @@ class TStorageConfigClient { ~TStorageConfigClient(); // Replace config - TAsyncStatus ReplaceStorageConfig( + TAsyncStatus ReplaceConfig( const std::optional& yaml_config, const std::optional& storage_yaml_config, - const TReplaceStorageConfigSettings& settings = {}); + const TReplaceConfigSettings& settings = {}); // Fetch current cluster storage config - TAsyncFetchStorageConfigResult FetchStorageConfig(bool dedicated_storage_section, bool dedicated_cluster_section, - const TFetchStorageConfigSettings& settings = {}); + TAsyncFetchConfigResult FetchConfig(bool dedicated_storage_section, bool dedicated_cluster_section, + const TFetchConfigSettings& settings = {}); // Bootstrap cluster with automatic configuration TAsyncStatus BootstrapCluster(const std::string& selfAssemblyUUID, const TBootstrapClusterSettings& settings = {}); diff --git a/ydb/public/sdk/cpp/src/client/bsconfig/storage_config.cpp b/ydb/public/sdk/cpp/src/client/bsconfig/storage_config.cpp index 4207b9c30149..085b890a001e 100644 --- a/ydb/public/sdk/cpp/src/client/bsconfig/storage_config.cpp +++ b/ydb/public/sdk/cpp/src/client/bsconfig/storage_config.cpp @@ -14,12 +14,12 @@ class TStorageConfigClient::TImpl : public TClientImplCommon& yaml_config, const std::optional& storage_yaml_config, - const TReplaceStorageConfigSettings& settings) + const TReplaceConfigSettings& settings) { - auto request = MakeRequest(); + auto request = MakeRequest(); if (yaml_config) { request.set_main_config(*yaml_config); @@ -40,41 +40,41 @@ class TStorageConfigClient::TImpl : public TClientImplCommon( + return RunSimple( std::move(request), - &Ydb::BSConfig::V1::BSConfigService::Stub::AsyncReplaceStorageConfig, + &Ydb::BSConfig::V1::BSConfigService::Stub::AsyncReplaceConfig, TRpcRequestSettings::Make(settings)); } - TAsyncFetchStorageConfigResult FetchStorageConfig(bool dedicated_storage_section, bool dedicated_cluster_section, - const TFetchStorageConfigSettings& settings) + TAsyncFetchConfigResult FetchConfig(bool dedicated_storage_section, bool dedicated_cluster_section, + const TFetchConfigSettings& settings) { - auto request = MakeOperationRequest(settings); + auto request = MakeOperationRequest(settings); if (dedicated_storage_section) { request.set_dedicated_storage_section(true); } if (dedicated_cluster_section) { request.set_dedicated_cluster_section(true); } - auto promise = NThreading::NewPromise(); + auto promise = NThreading::NewPromise(); auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { NYdb::TStringType config; NYdb::TStringType storage_config; - if (Ydb::BSConfig::FetchStorageConfigResult result; any && any->UnpackTo(&result)) { + if (Ydb::BSConfig::FetchConfigResult result; any && any->UnpackTo(&result)) { config = result.main_config(); storage_config = result.storage_config(); } - TFetchStorageConfigResult val(TStatus(std::move(status)), std::string{std::move(config)}, + TFetchConfigResult val(TStatus(std::move(status)), std::string{std::move(config)}, std::string{std::move(storage_config)}); promise.SetValue(std::move(val)); }; - Connections_->RunDeferred( + Connections_->RunDeferred( std::move(request), extractor, - &Ydb::BSConfig::V1::BSConfigService::Stub::AsyncFetchStorageConfig, + &Ydb::BSConfig::V1::BSConfigService::Stub::AsyncFetchConfig, DbDriverState_, INITIAL_DEFERRED_CALL_DELAY, TRpcRequestSettings::Make(settings)); @@ -97,18 +97,18 @@ TStorageConfigClient::TStorageConfigClient(const TDriver& driver, const TCommonC TStorageConfigClient::~TStorageConfigClient() = default; -TAsyncStatus TStorageConfigClient::ReplaceStorageConfig( +TAsyncStatus TStorageConfigClient::ReplaceConfig( const std::optional& yaml_config, const std::optional& storage_yaml_config, - const TReplaceStorageConfigSettings& settings) + const TReplaceConfigSettings& settings) { - return Impl_->ReplaceStorageConfig(yaml_config, storage_yaml_config, settings); + return Impl_->ReplaceConfig(yaml_config, storage_yaml_config, settings); } -TAsyncFetchStorageConfigResult TStorageConfigClient::FetchStorageConfig(bool dedicated_storage_section, - bool dedicated_cluster_section, const TFetchStorageConfigSettings& settings) +TAsyncFetchConfigResult TStorageConfigClient::FetchConfig(bool dedicated_storage_section, + bool dedicated_cluster_section, const TFetchConfigSettings& settings) { - return Impl_->FetchStorageConfig(dedicated_storage_section, dedicated_cluster_section, settings); + return Impl_->FetchConfig(dedicated_storage_section, dedicated_cluster_section, settings); } TAsyncStatus TStorageConfigClient::BootstrapCluster(const std::string& selfAssemblyUUID, const TBootstrapClusterSettings& settings) { diff --git a/ydb/services/bsconfig/grpc_service.cpp b/ydb/services/bsconfig/grpc_service.cpp index 3853531e0d6c..2a929982736e 100644 --- a/ydb/services/bsconfig/grpc_service.cpp +++ b/ydb/services/bsconfig/grpc_service.cpp @@ -28,8 +28,8 @@ void TBSConfigGRpcService::SetupIncomingRequests(NYdbGrpc::TLoggerPtr logger) { #define SETUP_BS_METHOD(methodName, method, rlMode, requestType) \ SETUP_METHOD(methodName, method, rlMode, requestType, BSConfig, bsconfig) - SETUP_BS_METHOD(ReplaceStorageConfig, DoReplaceBSConfig, Rps, BSCONFIG_REPLACESTORAGECONFIG); - SETUP_BS_METHOD(FetchStorageConfig, DoFetchBSConfig, Rps, BSCONFIG_FETCHSTORAGECONFIG); + SETUP_BS_METHOD(ReplaceConfig, DoReplaceBSConfig, Rps, BSCONFIG_REPLACESTORAGECONFIG); + SETUP_BS_METHOD(FetchConfig, DoFetchBSConfig, Rps, BSCONFIG_FETCHSTORAGECONFIG); SETUP_BS_METHOD(BootstrapCluster, DoBootstrapCluster, Rps, BSCONFIG_BOOTSTRAP); #undef SETUP_BS_METHOD From c64da477d6f7d7b5eacf09a4075c7de367bbe37b Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Thu, 13 Feb 2025 10:16:50 +0000 Subject: [PATCH 05/50] WIP --- ydb/core/driver_lib/run/run.cpp | 8 ++--- ydb/core/grpc_services/rpc_bsconfig.cpp | 32 +++++++++---------- ydb/core/grpc_services/rpc_bsconfig_base.h | 12 +++---- .../yaml_config/yaml_config_parser.cpp | 4 +-- ydb/library/yaml_config/yaml_config_parser.h | 2 +- ydb/public/api/grpc/ydb_bsconfig_v1.proto | 12 +++---- ydb/public/api/protos/ydb_bsconfig.proto | 4 +-- .../ydb_bsconfig/ydb_storage_config.cpp | 22 ++++++------- .../src/client/bsconfig/storage_config.cpp | 22 ++++++------- ydb/services/bsconfig/bsconfig_ut.cpp | 20 ++++++------ ydb/services/bsconfig/grpc_service.cpp | 10 +++--- ydb/services/bsconfig/grpc_service.h | 10 +++--- 12 files changed, 79 insertions(+), 79 deletions(-) diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index fc111a854c1e..5835a8e5a861 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -620,8 +620,8 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) { names["tablet_service"] = &hasTabletService; TServiceCfg hasView = services.empty(); names["view"] = &hasView; - TServiceCfg hasBSConfig = services.empty(); - names["bsconfig"] = &hasBSConfig; + TServiceCfg hasConfig = services.empty(); + names["config"] = &hasConfig; std::unordered_set enabled; for (const auto& name : services) { @@ -915,8 +915,8 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) { grpcRequestProxies[0], hasView.IsRlAllowed())); } - if (hasBSConfig) { - server.AddService(new NGRpcService::TBSConfigGRpcService(ActorSystem.Get(), Counters, grpcRequestProxies[0])); + if (hasConfig) { + server.AddService(new NGRpcService::TConfigGRpcService(ActorSystem.Get(), Counters, grpcRequestProxies[0])); } if (ModuleFactories) { diff --git a/ydb/core/grpc_services/rpc_bsconfig.cpp b/ydb/core/grpc_services/rpc_bsconfig.cpp index 6ea35e62f40b..24f1730de216 100644 --- a/ydb/core/grpc_services/rpc_bsconfig.cpp +++ b/ydb/core/grpc_services/rpc_bsconfig.cpp @@ -12,33 +12,33 @@ namespace NKikimr::NGRpcService { using TEvReplaceStorageConfigRequest = - TGrpcRequestOperationCall; + TGrpcRequestOperationCall; using TEvFetchStorageConfigRequest = - TGrpcRequestOperationCall; + TGrpcRequestOperationCall; using TEvBootstrapClusterRequest = - TGrpcRequestOperationCall; + TGrpcRequestOperationCall; using namespace NActors; using namespace Ydb; -bool CopyToConfigRequest(const Ydb::BSConfig::ReplaceConfigRequest &from, NKikimrBlobStorage::TConfigRequest *to) { +bool CopyToConfigRequest(const Ydb::Config::ReplaceConfigRequest &from, NKikimrBlobStorage::TConfigRequest *to) { to->CopyFrom(NKikimr::NYaml::BuildInitDistributedStorageCommand(from.main_config())); return true; } -void CopyFromConfigResponse(const NKikimrBlobStorage::TConfigResponse &/*from*/, Ydb::BSConfig::ReplaceConfigResult* /*to*/) { +void CopyFromConfigResponse(const NKikimrBlobStorage::TConfigResponse &/*from*/, Ydb::Config::ReplaceConfigResult* /*to*/) { } -bool CopyToConfigRequest(const Ydb::BSConfig::FetchConfigRequest &/*from*/, NKikimrBlobStorage::TConfigRequest *to) { +bool CopyToConfigRequest(const Ydb::Config::FetchConfigRequest &/*from*/, NKikimrBlobStorage::TConfigRequest *to) { to->AddCommand()->MutableReadHostConfig(); to->AddCommand()->MutableReadBox(); return true; } -void CopyFromConfigResponse(const NKikimrBlobStorage::TConfigResponse &from, Ydb::BSConfig::FetchConfigResult *to) { +void CopyFromConfigResponse(const NKikimrBlobStorage::TConfigResponse &from, Ydb::Config::FetchConfigResult *to) { auto hostConfigStatus = from.GetStatus()[0]; auto boxStatus = from.GetStatus()[1]; NKikimrConfig::StorageConfig storageConfig; @@ -81,9 +81,9 @@ void CopyFromConfigResponse(const NKikimrBlobStorage::TConfigResponse &from, Ydb } class TReplaceStorageConfigRequest : public TBSConfigRequestGrpc { + Ydb::Config::ReplaceConfigResult> { public: - using TBase = TBSConfigRequestGrpc; + using TBase = TBSConfigRequestGrpc; using TBase::TBase; bool ValidateRequest(Ydb::StatusIds::StatusCode& /*status*/, NYql::TIssues& /*issues*/) override { @@ -99,7 +99,7 @@ class TReplaceStorageConfigRequest : public TBSConfigRequestGrpc { + Ydb::Config::FetchConfigResult> { public: - using TBase = TBSConfigRequestGrpc; + using TBase = TBSConfigRequestGrpc; using TBase::TBase; bool ValidateRequest(Ydb::StatusIds::StatusCode& /*status*/, NYql::TIssues& /*issues*/) override { @@ -147,7 +147,7 @@ class TFetchStorageConfigRequest : public TBSConfigRequestGrpc class TBaseBSConfigRequest { @@ -281,7 +281,7 @@ class TBSConfigRequestGrpc : public TRpcOperationRequestActor) { + if constexpr (std::is_same_v) { TResultRecord result; const auto& record = ev->Get()->Record; if (record.HasClusterYaml()) { @@ -299,7 +299,7 @@ class TBSConfigRequestGrpc : public TRpcOperationRequestActor) { + if constexpr (std::is_same_v) { const auto& record = ev->Get()->Record; if (record.GetStatus() == NKikimrBlobStorage::TEvControllerReplaceConfigResponse::Success) { TResultRecord result; diff --git a/ydb/library/yaml_config/yaml_config_parser.cpp b/ydb/library/yaml_config/yaml_config_parser.cpp index ead77a38d1ec..4f7914261059 100644 --- a/ydb/library/yaml_config/yaml_config_parser.cpp +++ b/ydb/library/yaml_config/yaml_config_parser.cpp @@ -1483,8 +1483,8 @@ namespace NKikimr::NYaml { return result; } - Ydb::BSConfig::ReplaceConfigRequest BuildReplaceDistributedStorageCommand(const TString& data) { - Ydb::BSConfig::ReplaceConfigRequest replaceRequest; + Ydb::Config::ReplaceConfigRequest BuildReplaceDistributedStorageCommand(const TString& data) { + Ydb::Config::ReplaceConfigRequest replaceRequest; replaceRequest.set_main_config(data); return replaceRequest; } diff --git a/ydb/library/yaml_config/yaml_config_parser.h b/ydb/library/yaml_config/yaml_config_parser.h index 2b3b7419e53b..7e1928f4c96e 100644 --- a/ydb/library/yaml_config/yaml_config_parser.h +++ b/ydb/library/yaml_config/yaml_config_parser.h @@ -56,7 +56,7 @@ namespace NKikimr::NYaml { TSimpleSharedPtr unknownFieldsCollector = nullptr); NKikimrBlobStorage::TConfigRequest BuildInitDistributedStorageCommand(const TString& data); - Ydb::BSConfig::ReplaceConfigRequest BuildReplaceDistributedStorageCommand(const TString& data); + Ydb::Config::ReplaceConfigRequest BuildReplaceDistributedStorageCommand(const TString& data); TString ParseProtoToYaml(const NKikimrConfig::StorageConfig& protoConfig); void ExtractExtraFields(NJson::TJsonValue& json, TTransformContext& ctx); diff --git a/ydb/public/api/grpc/ydb_bsconfig_v1.proto b/ydb/public/api/grpc/ydb_bsconfig_v1.proto index ef8d915aa5b1..d8861ae9d0e9 100644 --- a/ydb/public/api/grpc/ydb_bsconfig_v1.proto +++ b/ydb/public/api/grpc/ydb_bsconfig_v1.proto @@ -1,22 +1,22 @@ syntax = "proto3"; -package Ydb.BSConfig.V1; +package Ydb.Config.V1; option java_package = "com.yandex.ydb.bsconfig.v1"; -option java_outer_classname = "BSConfigGrpc"; +option java_outer_classname = "ConfigGrpc"; option java_multiple_files = true; import "ydb/public/api/protos/ydb_bsconfig.proto"; -service BSConfigService { +service ConfigService { // Initialize Blobstorage/single config - rpc ReplaceConfig(BSConfig.ReplaceConfigRequest) returns (BSConfig.ReplaceConfigResponse); + rpc ReplaceConfig(Config.ReplaceConfigRequest) returns (Config.ReplaceConfigResponse); // Fetch Blobstorage/single config - rpc FetchConfig(BSConfig.FetchConfigRequest) returns (BSConfig.FetchConfigResponse); + rpc FetchConfig(Config.FetchConfigRequest) returns (Config.FetchConfigResponse); // Bootstrap automatically configured cluster - rpc BootstrapCluster(BSConfig.BootstrapClusterRequest) returns (BSConfig.BootstrapClusterResponse); + rpc BootstrapCluster(Config.BootstrapClusterRequest) returns (Config.BootstrapClusterResponse); } diff --git a/ydb/public/api/protos/ydb_bsconfig.proto b/ydb/public/api/protos/ydb_bsconfig.proto index 9888b589376a..f7ab5e45c4b8 100644 --- a/ydb/public/api/protos/ydb_bsconfig.proto +++ b/ydb/public/api/protos/ydb_bsconfig.proto @@ -1,10 +1,10 @@ syntax = "proto3"; option cc_enable_arenas = true; -package Ydb.BSConfig; +package Ydb.Config; option java_package = "com.yandex.ydb.bsconfig.proto"; -option java_outer_classname = "BSConfigProtos"; +option java_outer_classname = "ConfigProtos"; option java_multiple_files = true; import "ydb/public/api/protos/ydb_operation.proto"; diff --git a/ydb/public/sdk/cpp/client/ydb_bsconfig/ydb_storage_config.cpp b/ydb/public/sdk/cpp/client/ydb_bsconfig/ydb_storage_config.cpp index c4d588aad92b..bf03e9a5ff00 100644 --- a/ydb/public/sdk/cpp/client/ydb_bsconfig/ydb_storage_config.cpp +++ b/ydb/public/sdk/cpp/client/ydb_bsconfig/ydb_storage_config.cpp @@ -13,21 +13,21 @@ class TStorageConfigClient::TImpl : public TClientImplCommon(); + auto request = MakeRequest(); request.set_yaml_config(config); - return RunSimple( + return RunSimple( std::move(request), - &Ydb::BSConfig::V1::BSConfigService::Stub::AsyncReplaceStorageConfig); + &Ydb::Config::V1::BSConfigService::Stub::AsyncReplaceStorageConfig); } TAsyncFetchStorageConfigResult FetchStorageConfig(const TStorageConfigSettings& settings = {}) { - auto request = MakeOperationRequest(settings); + auto request = MakeOperationRequest(settings); auto promise = NThreading::NewPromise(); auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { TString config; - if (Ydb::BSConfig::FetchStorageConfigResult result; any && any->UnpackTo(&result)) { + if (Ydb::Config::FetchStorageConfigResult result; any && any->UnpackTo(&result)) { config = result.yaml_config(); } @@ -35,10 +35,10 @@ class TStorageConfigClient::TImpl : public TClientImplCommonRunDeferred( + Connections_->RunDeferred( std::move(request), extractor, - &Ydb::BSConfig::V1::BSConfigService::Stub::AsyncFetchStorageConfig, + &Ydb::Config::V1::BSConfigService::Stub::AsyncFetchStorageConfig, DbDriverState_, INITIAL_DEFERRED_CALL_DELAY, TRpcRequestSettings::Make(settings)); @@ -46,12 +46,12 @@ class TStorageConfigClient::TImpl : public TClientImplCommon(); + auto request = MakeRequest(); request.set_self_assembly_uuid(selfAssemblyUUID); - return RunSimple(std::move(request), - &Ydb::BSConfig::V1::BSConfigService::Stub::AsyncBootstrapCluster); + return RunSimple(std::move(request), + &Ydb::Config::V1::BSConfigService::Stub::AsyncBootstrapCluster); } }; diff --git a/ydb/public/sdk/cpp/src/client/bsconfig/storage_config.cpp b/ydb/public/sdk/cpp/src/client/bsconfig/storage_config.cpp index 085b890a001e..6b25657e400e 100644 --- a/ydb/public/sdk/cpp/src/client/bsconfig/storage_config.cpp +++ b/ydb/public/sdk/cpp/src/client/bsconfig/storage_config.cpp @@ -19,7 +19,7 @@ class TStorageConfigClient::TImpl : public TClientImplCommon& storage_yaml_config, const TReplaceConfigSettings& settings) { - auto request = MakeRequest(); + auto request = MakeRequest(); if (yaml_config) { request.set_main_config(*yaml_config); @@ -40,16 +40,16 @@ class TStorageConfigClient::TImpl : public TClientImplCommon( + return RunSimple( std::move(request), - &Ydb::BSConfig::V1::BSConfigService::Stub::AsyncReplaceConfig, + &Ydb::Config::V1::ConfigService::Stub::AsyncReplaceConfig, TRpcRequestSettings::Make(settings)); } TAsyncFetchConfigResult FetchConfig(bool dedicated_storage_section, bool dedicated_cluster_section, const TFetchConfigSettings& settings) { - auto request = MakeOperationRequest(settings); + auto request = MakeOperationRequest(settings); if (dedicated_storage_section) { request.set_dedicated_storage_section(true); } @@ -61,7 +61,7 @@ class TStorageConfigClient::TImpl : public TClientImplCommonUnpackTo(&result)) { + if (Ydb::Config::FetchConfigResult result; any && any->UnpackTo(&result)) { config = result.main_config(); storage_config = result.storage_config(); } @@ -71,10 +71,10 @@ class TStorageConfigClient::TImpl : public TClientImplCommonRunDeferred( + Connections_->RunDeferred( std::move(request), extractor, - &Ydb::BSConfig::V1::BSConfigService::Stub::AsyncFetchConfig, + &Ydb::Config::V1::ConfigService::Stub::AsyncFetchConfig, DbDriverState_, INITIAL_DEFERRED_CALL_DELAY, TRpcRequestSettings::Make(settings)); @@ -82,11 +82,11 @@ class TStorageConfigClient::TImpl : public TClientImplCommon(); + auto request = MakeRequest(); request.set_self_assembly_uuid(selfAssemblyUUID); - return RunSimple(std::move(request), - &Ydb::BSConfig::V1::BSConfigService::Stub::AsyncBootstrapCluster, + return RunSimple(std::move(request), + &Ydb::Config::V1::ConfigService::Stub::AsyncBootstrapCluster, TRpcRequestSettings::Make(settings)); } }; diff --git a/ydb/services/bsconfig/bsconfig_ut.cpp b/ydb/services/bsconfig/bsconfig_ut.cpp index 1cb4aa3696dd..eff2f5a763bb 100644 --- a/ydb/services/bsconfig/bsconfig_ut.cpp +++ b/ydb/services/bsconfig/bsconfig_ut.cpp @@ -167,10 +167,10 @@ Y_UNIT_TEST_SUITE(BSConfigGRPCService) { void ReplaceStorageConfig(auto &channel, std::optional yamlConfig, std::optional storageYamlConfig, std::optional switchDedicatedStorageSection, bool dedicatedConfigMode) { - std::unique_ptr stub; - stub = Ydb::BSConfig::V1::BSConfigService::NewStub(channel); + std::unique_ptr stub; + stub = Ydb::Config::V1::BSConfigService::NewStub(channel); - Ydb::BSConfig::ReplaceStorageConfigRequest request; + Ydb::Config::ReplaceStorageConfigRequest request; if (yamlConfig) { request.set_yaml_config(*yamlConfig); } @@ -182,8 +182,8 @@ Y_UNIT_TEST_SUITE(BSConfigGRPCService) { } request.set_dedicated_config_mode(dedicatedConfigMode); - Ydb::BSConfig::ReplaceStorageConfigResponse response; - Ydb::BSConfig::ReplaceStorageConfigResult result; + Ydb::Config::ReplaceStorageConfigResponse response; + Ydb::Config::ReplaceStorageConfigResult result; grpc::ClientContext replaceStorageConfigCtx; AdjustCtxForDB(replaceStorageConfigCtx); @@ -195,15 +195,15 @@ Y_UNIT_TEST_SUITE(BSConfigGRPCService) { void FetchStorageConfig(auto& channel, bool dedicatedStorageSection, bool dedicatedClusterSection, std::optional& yamlConfig, std::optional& storageYamlConfig) { - std::unique_ptr stub; - stub = Ydb::BSConfig::V1::BSConfigService::NewStub(channel); + std::unique_ptr stub; + stub = Ydb::Config::V1::BSConfigService::NewStub(channel); - Ydb::BSConfig::FetchStorageConfigRequest request; + Ydb::Config::FetchStorageConfigRequest request; request.set_dedicated_storage_section(dedicatedStorageSection); request.set_dedicated_cluster_section(dedicatedClusterSection); - Ydb::BSConfig::FetchStorageConfigResponse response; - Ydb::BSConfig::FetchStorageConfigResult result; + Ydb::Config::FetchStorageConfigResponse response; + Ydb::Config::FetchStorageConfigResult result; grpc::ClientContext fetchStorageConfigCtx; AdjustCtxForDB(fetchStorageConfigCtx); diff --git a/ydb/services/bsconfig/grpc_service.cpp b/ydb/services/bsconfig/grpc_service.cpp index 2a929982736e..4e7e257c8169 100644 --- a/ydb/services/bsconfig/grpc_service.cpp +++ b/ydb/services/bsconfig/grpc_service.cpp @@ -8,25 +8,25 @@ namespace NKikimr::NGRpcService { -TBSConfigGRpcService::TBSConfigGRpcService(NActors::TActorSystem* actorSystem, TIntrusivePtr counters, NActors::TActorId grpcRequestProxyId) \ +TConfigGRpcService::TConfigGRpcService(NActors::TActorSystem* actorSystem, TIntrusivePtr counters, NActors::TActorId grpcRequestProxyId) \ : ActorSystem(actorSystem) \ , Counters(std::move(counters)) , GRpcRequestProxyId(grpcRequestProxyId) { } -TBSConfigGRpcService::~TBSConfigGRpcService() = default; +TConfigGRpcService::~TConfigGRpcService() = default; -void TBSConfigGRpcService::InitService(grpc::ServerCompletionQueue* cq, NYdbGrpc::TLoggerPtr logger) { +void TConfigGRpcService::InitService(grpc::ServerCompletionQueue* cq, NYdbGrpc::TLoggerPtr logger) { CQ = cq; SetupIncomingRequests(std::move(logger)); } -void TBSConfigGRpcService::SetupIncomingRequests(NYdbGrpc::TLoggerPtr logger) { +void TConfigGRpcService::SetupIncomingRequests(NYdbGrpc::TLoggerPtr logger) { auto getCounterBlock = NGRpcService::CreateCounterCb(Counters, ActorSystem); #define SETUP_BS_METHOD(methodName, method, rlMode, requestType) \ - SETUP_METHOD(methodName, method, rlMode, requestType, BSConfig, bsconfig) + SETUP_METHOD(methodName, method, rlMode, requestType, Config, bsconfig) SETUP_BS_METHOD(ReplaceConfig, DoReplaceBSConfig, Rps, BSCONFIG_REPLACESTORAGECONFIG); SETUP_BS_METHOD(FetchConfig, DoFetchBSConfig, Rps, BSCONFIG_FETCHSTORAGECONFIG); diff --git a/ydb/services/bsconfig/grpc_service.h b/ydb/services/bsconfig/grpc_service.h index 4506aa3f1334..18d95b2071e0 100644 --- a/ydb/services/bsconfig/grpc_service.h +++ b/ydb/services/bsconfig/grpc_service.h @@ -8,13 +8,13 @@ namespace NKikimr::NGRpcService { -class TBSConfigGRpcService - : public NYdbGrpc::TGrpcServiceBase +class TConfigGRpcService + : public NYdbGrpc::TGrpcServiceBase { public: - TBSConfigGRpcService(NActors::TActorSystem* actorSystem, TIntrusivePtr counters, + TConfigGRpcService(NActors::TActorSystem* actorSystem, TIntrusivePtr counters, NActors::TActorId grpcRequestProxyId); - ~TBSConfigGRpcService(); + ~TConfigGRpcService(); void InitService(grpc::ServerCompletionQueue* cq, NYdbGrpc::TLoggerPtr logger) override; @@ -29,4 +29,4 @@ class TBSConfigGRpcService grpc::ServerCompletionQueue* CQ = nullptr; }; -} // namespace NKikimr::NGRpcService \ No newline at end of file +} // namespace NKikimr::NGRpcService From fe0e88df7309e8800ede0d27e0a3594cbd39ba87 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Fri, 14 Feb 2025 08:37:00 +0000 Subject: [PATCH 06/50] WIP --- ydb/core/grpc_services/rpc_bsconfig.cpp | 28 +++- ydb/core/grpc_services/rpc_bsconfig_base.h | 6 +- .../yaml_config/yaml_config_parser.cpp | 2 +- ydb/public/api/protos/ydb_bsconfig.proto | 145 ++++++++++++++---- 4 files changed, 146 insertions(+), 35 deletions(-) diff --git a/ydb/core/grpc_services/rpc_bsconfig.cpp b/ydb/core/grpc_services/rpc_bsconfig.cpp index 24f1730de216..42be31cb7e33 100644 --- a/ydb/core/grpc_services/rpc_bsconfig.cpp +++ b/ydb/core/grpc_services/rpc_bsconfig.cpp @@ -117,15 +117,33 @@ class TReplaceStorageConfigRequest : public TBSConfigRequestGrpc switch_dedicated_storage_section; + bool dedicated_config_mode = false; + switch (request->action_case()) { + case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceEnableDedicatedStorageSection: + switch_dedicated_storage_section = true; + dedicated_config_mode = true; + case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceDisableDedicatedStorageSection: + switch_dedicated_storage_section = false; + case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceWithDedicatedStorageSection: + dedicated_config_mode = true; + [[fallthrough]]; + case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplace: + break; + case Ydb::Config::ReplaceConfigRequest::ActionCase::ACTION_NOT_SET: + break; // TODO: handle as error? + } + using T = std::decay_t; return std::make_unique( opt(&T::has_main_config, &T::main_config), opt(&T::has_storage_config, &T::storage_config), - opt(&T::has_switch_dedicated_storage_section, &T::switch_dedicated_storage_section), - request->dedicated_config_mode(), - request->allow_unknown_fields(), - request->allow_incorrect_version(), - request->allow_incorrect_cluster()); + switch_dedicated_storage_section, + dedicated_config_mode, + request->allow_unknown_fields() || request->bypass_checks(), + request->bypass_checks(), + request->bypass_checks()); } }; diff --git a/ydb/core/grpc_services/rpc_bsconfig_base.h b/ydb/core/grpc_services/rpc_bsconfig_base.h index e987bc64cba0..c52869e4ac04 100644 --- a/ydb/core/grpc_services/rpc_bsconfig_base.h +++ b/ydb/core/grpc_services/rpc_bsconfig_base.h @@ -285,10 +285,12 @@ class TBSConfigRequestGrpc : public TRpcOperationRequestActorGet()->Record; if (record.HasClusterYaml()) { - result.set_main_config(ev->Get()->Record.GetClusterYaml()); + // FIXME: fill identity + result.add_config()->set_config(ev->Get()->Record.GetClusterYaml()); } if (record.HasStorageYaml()) { - result.set_storage_config(ev->Get()->Record.GetStorageYaml()); + // FIXME: fill identity + result.add_config()->set_config(ev->Get()->Record.GetStorageYaml()); } self->ReplyWithResult(Ydb::StatusIds::SUCCESS, result, self->ActorContext()); } else { diff --git a/ydb/library/yaml_config/yaml_config_parser.cpp b/ydb/library/yaml_config/yaml_config_parser.cpp index 4f7914261059..8cdc1032f387 100644 --- a/ydb/library/yaml_config/yaml_config_parser.cpp +++ b/ydb/library/yaml_config/yaml_config_parser.cpp @@ -1485,7 +1485,7 @@ namespace NKikimr::NYaml { Ydb::Config::ReplaceConfigRequest BuildReplaceDistributedStorageCommand(const TString& data) { Ydb::Config::ReplaceConfigRequest replaceRequest; - replaceRequest.set_main_config(data); + replaceRequest.mutable_replace()->add_config(data); return replaceRequest; } diff --git a/ydb/public/api/protos/ydb_bsconfig.proto b/ydb/public/api/protos/ydb_bsconfig.proto index f7ab5e45c4b8..759673a2d68c 100644 --- a/ydb/public/api/protos/ydb_bsconfig.proto +++ b/ydb/public/api/protos/ydb_bsconfig.proto @@ -3,67 +3,158 @@ option cc_enable_arenas = true; package Ydb.Config; -option java_package = "com.yandex.ydb.bsconfig.proto"; +option java_package = "com.yandex.ydb.config.proto"; option java_outer_classname = "ConfigProtos"; option java_multiple_files = true; import "ydb/public/api/protos/ydb_operation.proto"; +import "google/protobuf/empty.proto"; -// -// BSConfig API. -// +message MainConfigIdentity {} -message ReplaceConfigRequest { - Ydb.Operations.OperationParams operation_params = 1; - - // cluster yaml config - optional string main_config = 2; +message StorageConfigIdentity {} - // dedicated storage yaml config (when dual-config mode is enabled) - optional string storage_config = 3; - - // if filled, can turn on or off dedicated section of YAML config - optional bool switch_dedicated_storage_section = 4; +message DatabaseConfigIdentity { + string database = 1; +} - // if true, then user expects system to work in dual-config mode - bool dedicated_config_mode = 5; +// Representation of config metadata. +// Uniquely identifies config. +// Used to describe what config user want to change or receive. +message ConfigIdentity { + // Current config version + uint64 version = 1; + + // Identifier of the cluster + string cluster = 2; + + // Config type with attached identity data specific for particular type + // List of config types may be extended in future + oneof type { + MainConfigIdentity main = 3; + StorageConfigIdentity storage = 4; + DatabaseConfigIdentity database = 5; + } +} - bool dry_run = 7; +message ReplaceConfigRequest { + Ydb.Operations.OperationParams operation_params = 1; - bool allow_unknown_fields = 8; - bool allow_absent_database = 9; - // next two together allow to mimic DynConfigService.SetConfig behavior - bool allow_incorrect_version = 10; - bool allow_incorrect_cluster = 11; + // Zero or more configs of any type. + // Exact type of config identified by metadata stored inside config. + message ConfigBundle { + repeated string config = 1; + } + + oneof action { + // Replace config on cluster. + // By default awaits single config with matching metadata (kind MainConfig, right cluster and version + 1) + ConfigBundle replace = 2; + + // Replace storage and/or main configs on cluster in dedicated storage config mode + // By default awaits either MainConfig or StorageConfig or both with matching metadata + ConfigBundle replace_with_dedicated_storage_section = 3; + + // Replace storage and main configs on cluster in single config mode **AND** enables dedicated storage config mode + // By default awaits MainConfig and StorageConfig with matching metadata + ConfigBundle replace_enable_dedicated_storage_section = 4; + + // Replace main config on cluster in single config mode **AND** disables dedicated storage config mode + // By default awaits MainConfig with matching metadata + ConfigBundle replace_disable_dedicated_storage_section = 5; + } + + // Try to apply config(s) without making real changes + // Exact data returned and checks done are implementation defined + // Now renders final configs for all labels and makes basic validity checks + // All errors returned as issues in response + bool dry_run = 6; + + // Allows to apply config with fields unknown to out parser + // It is useful when user need to try some new experimental flag, then rollback to previous version + // and want to update existing config keeping flag from newer version in config + // All unknown fields still reported in result as warnings + bool allow_unknown_fields = 7; + + // Allows to apply config with incorrect metadata + // Metadata of stored config after this call will be replaced with correct one + // Exact set of excluded checks is implementation defined, this flag + // may ignore additional checks in future + // BEWARE: with this flag set user can apply config on wrong cluster + // or overwrite other users changes in case of race + // Behaviour identical to deprecated DynConfig.SetConfig() call + bool bypass_checks = 8; } message ReplaceConfigResponse { Ydb.Operations.Operation operation = 1; } -message ReplaceConfigResult { +message ReplaceConfigResult {} + +message FetchModeAll { + // Use this option to somehow transform main config + // Currently used to automatically derive detached StorageSection from MainConfig + // or vice versa + oneof config_transform { + // Optionally may be set to explicitly tell "do not transform anything" + google.protobuf.Empty none = 1; + + // Fetch will return MainConfig and StorageConfig derived from MainConfig + // by detaching storage config + google.protobuf.Empty detach_storage_config_section = 2; + + // Fetch will return MainConfig derived from MainConfig and StorageConfig + // by attaching storage config to main config + google.protobuf.Empty attach_storage_config_section = 3; + } +} + +message FetchModeByIdentity { + // List of configs to fetch + repeated ConfigIdentity identity = 1; } message FetchConfigRequest { Ydb.Operations.OperationParams operation_params = 1; - bool dedicated_storage_section = 2; - bool dedicated_cluster_section = 3; + + // Depending on mode Fetch will return different set of configs + // List of modes may be extended in future + oneof mode { + // Fetch all configs from cluster + FetchModeAll all = 2; + + // Fetch all configs with matching identities + FetchModeByIdentity target = 3; + } } message FetchConfigResponse { Ydb.Operations.Operation operation = 1; } +// Config with attached identity +// Allows to operate on on configs without necessary parsing config metadta field +// For example filtering out fetch result based on some specific rules +message ConfigEntry { + ConfigIdentity identity = 1; + string config = 2; +} + message FetchConfigResult { - optional string main_config = 1; - optional string storage_config = 2; + repeated ConfigEntry config = 1; } message BootstrapClusterRequest { Ydb.Operations.OperationParams operation_params = 1; + + // This field used to make this request indempotent + // Must be set to arbitrary **unique** string string self_assembly_uuid = 2; } message BootstrapClusterResponse { Ydb.Operations.Operation operation = 1; } + +message BootstrapClusterResult {} From f2e26b76a0c054a3c6305b7004b6e0a746ca807c Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Fri, 14 Feb 2025 09:47:17 +0000 Subject: [PATCH 07/50] WIP --- ydb/core/grpc_services/rpc_bsconfig.cpp | 57 +++++++++++++++++-------- 1 file changed, 40 insertions(+), 17 deletions(-) diff --git a/ydb/core/grpc_services/rpc_bsconfig.cpp b/ydb/core/grpc_services/rpc_bsconfig.cpp index 42be31cb7e33..24a425053efa 100644 --- a/ydb/core/grpc_services/rpc_bsconfig.cpp +++ b/ydb/core/grpc_services/rpc_bsconfig.cpp @@ -25,7 +25,9 @@ using namespace NActors; using namespace Ydb; bool CopyToConfigRequest(const Ydb::Config::ReplaceConfigRequest &from, NKikimrBlobStorage::TConfigRequest *to) { - to->CopyFrom(NKikimr::NYaml::BuildInitDistributedStorageCommand(from.main_config())); + Y_UNUSED(to, from); + // FIXME + // to->CopyFrom(NKikimr::NYaml::BuildInitDistributedStorageCommand(from.main_config())); return true; } @@ -77,7 +79,9 @@ void CopyFromConfigResponse(const NKikimrBlobStorage::TConfigResponse &from, Ydb } } storageConfig.set_item_config_generation(itemConfigGeneration); - to->set_main_config(NYaml::ParseProtoToYaml(storageConfig)); + Y_UNUSED(to); + // FIXME: + // to->set_main_config(NYaml::ParseProtoToYaml(storageConfig)); } class TReplaceStorageConfigRequest : public TBSConfigRequestGrpcSetYAML(GetProtoRequest()->main_config()); + Y_UNUSED(cmd); + // FIXME + // cmd->SetYAML(GetProtoRequest()->main_config()); } void FillDistconfResult(NKikimrBlobStorage::TEvNodeConfigInvokeOnRootResult& /*record*/, @@ -105,7 +111,8 @@ class TReplaceStorageConfigRequest : public TBSConfigRequestGrpcmain_config()); + // FIXME: + // newConfig = NYaml::Parse(GetProtoRequest()->main_config()); } catch (const std::exception&) { return false; // assuming no distconf enabled in this config } @@ -114,31 +121,42 @@ class TReplaceStorageConfigRequest : public TBSConfigRequestGrpc ProcessControllerQuery() override { auto *request = GetProtoRequest(); - auto opt = [&](auto&& has, auto&& get) { - return std::invoke(has, request) ? std::make_optional(std::invoke(get, request)) : std::nullopt; - }; std::optional switch_dedicated_storage_section; + std::optional mainConfig; + std::optional storageConfig; bool dedicated_config_mode = false; + + auto fillConfigs = [&](const auto& configBundle) { + for (const auto& config : configBundle.config()) { + Y_UNUSED(config); + } + }; + switch (request->action_case()) { case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceEnableDedicatedStorageSection: switch_dedicated_storage_section = true; dedicated_config_mode = true; + fillConfigs(request->replace_enable_dedicated_storage_section()); + break; case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceDisableDedicatedStorageSection: switch_dedicated_storage_section = false; + fillConfigs(request->replace_enable_dedicated_storage_section()); + break; case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceWithDedicatedStorageSection: dedicated_config_mode = true; + fillConfigs(request->replace_enable_dedicated_storage_section()); [[fallthrough]]; case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplace: + fillConfigs(request->replace_enable_dedicated_storage_section()); break; case Ydb::Config::ReplaceConfigRequest::ActionCase::ACTION_NOT_SET: break; // TODO: handle as error? } - using T = std::decay_t; return std::make_unique( - opt(&T::has_main_config, &T::main_config), - opt(&T::has_storage_config, &T::storage_config), + mainConfig, + storageConfig, switch_dedicated_storage_section, dedicated_config_mode, request->allow_unknown_fields() || request->bypass_checks(), @@ -166,7 +184,9 @@ class TFetchStorageConfigRequest : public TBSConfigRequestGrpc ProcessControllerQuery() override { - auto& request = *GetProtoRequest(); - auto ev = std::make_unique(); - auto& record = ev->Record; - record.SetDedicatedStorageSection(request.dedicated_storage_section()); - record.SetDedicatedClusterSection(request.dedicated_cluster_section()); - return ev; + // FIXME: + // auto& request = *GetProtoRequest(); + // auto ev = std::make_unique(); + // auto& record = ev->Record; + + // record.SetDedicatedStorageSection(request.dedicated_storage_section()); + // record.SetDedicatedClusterSection(request.dedicated_cluster_section()); + // return ev; + return nullptr; // FIXME: remove } }; From fa3f6573763f6513715104bc03fc15a7f0c52d93 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Fri, 14 Feb 2025 11:42:40 +0000 Subject: [PATCH 08/50] WIP --- ydb/core/grpc_services/rpc_bsconfig.cpp | 60 ++++++++++++++++++------- 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/ydb/core/grpc_services/rpc_bsconfig.cpp b/ydb/core/grpc_services/rpc_bsconfig.cpp index 24a425053efa..5dfdfb8a286b 100644 --- a/ydb/core/grpc_services/rpc_bsconfig.cpp +++ b/ydb/core/grpc_services/rpc_bsconfig.cpp @@ -1,6 +1,7 @@ #include "service_keyvalue.h" #include #include +#include #include "rpc_bsconfig_base.h" #include @@ -79,9 +80,14 @@ void CopyFromConfigResponse(const NKikimrBlobStorage::TConfigResponse &from, Ydb } } storageConfig.set_item_config_generation(itemConfigGeneration); - Y_UNUSED(to); - // FIXME: - // to->set_main_config(NYaml::ParseProtoToYaml(storageConfig)); + auto& config = *to->add_config(); + auto& identity = *config.mutable_identity(); + identity.set_version(itemConfigGeneration); + identity.set_cluster(AppData()->ClusterName); + identity.mutable_storage(); + // TODO: !imp fill metadata ? + // are we sure that it is storage config? + config.set_config(NYaml::ParseProtoToYaml(storageConfig)); } class TReplaceStorageConfigRequest : public TBSConfigRequestGrpcClusterName); + identity.mutable_main(); // TODO: is it really main ? + config.set_config(conf); } bool IsDistconfEnableQuery() const { @@ -194,15 +212,25 @@ class TFetchStorageConfigRequest : public TBSConfigRequestGrpc ProcessControllerQuery() override { - // FIXME: - // auto& request = *GetProtoRequest(); - // auto ev = std::make_unique(); - // auto& record = ev->Record; - - // record.SetDedicatedStorageSection(request.dedicated_storage_section()); - // record.SetDedicatedClusterSection(request.dedicated_cluster_section()); - // return ev; - return nullptr; // FIXME: remove + auto& request = *GetProtoRequest(); + auto ev = std::make_unique(); + auto& record = ev->Record; + + switch (request.mode_case()) { + case Ydb::Config::FetchConfigRequest::ModeCase::kAll: + if (request.all().config_transform_case() == Ydb::Config::FetchModeAll::ConfigTransformCase::kDetachStorageConfigSection) { + record.SetDedicatedStorageSection(true); + record.SetDedicatedClusterSection(true); + } + break; + case Ydb::Config::FetchConfigRequest::ModeCase::kTarget: + // FIXME: error + break; + case Ydb::Config::FetchConfigRequest::ModeCase::MODE_NOT_SET: + break; // TODO: maybe error + } + + return ev; } }; From f3ed2383421cdbca8ebcbd64e109cbf137ac6d92 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Fri, 14 Feb 2025 13:55:24 +0000 Subject: [PATCH 09/50] WIP --- ydb/core/grpc_services/rpc_bsconfig.cpp | 73 ++++++++++++++++--- ydb/core/grpc_services/rpc_bsconfig_base.h | 22 +++++- .../yaml_config/public/yaml_config.cpp | 13 ++++ ydb/library/yaml_config/public/yaml_config.h | 13 ++++ 4 files changed, 106 insertions(+), 15 deletions(-) diff --git a/ydb/core/grpc_services/rpc_bsconfig.cpp b/ydb/core/grpc_services/rpc_bsconfig.cpp index 5dfdfb8a286b..50aaf89062de 100644 --- a/ydb/core/grpc_services/rpc_bsconfig.cpp +++ b/ydb/core/grpc_services/rpc_bsconfig.cpp @@ -26,9 +26,34 @@ using namespace NActors; using namespace Ydb; bool CopyToConfigRequest(const Ydb::Config::ReplaceConfigRequest &from, NKikimrBlobStorage::TConfigRequest *to) { - Y_UNUSED(to, from); - // FIXME - // to->CopyFrom(NKikimr::NYaml::BuildInitDistributedStorageCommand(from.main_config())); + TString configStr; + + auto fillConfigs = [&](const auto& configBundle) { + for (const auto& config : configBundle.config()) { + if (NYamlConfig::IsMainConfig(config)) { + configStr = config; + } + } + }; + + switch (from.action_case()) { + case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceEnableDedicatedStorageSection: + fillConfigs(from.replace_enable_dedicated_storage_section()); + break; + case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceDisableDedicatedStorageSection: + fillConfigs(from.replace_disable_dedicated_storage_section()); + break; + case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceWithDedicatedStorageSection: + fillConfigs(from.replace_with_dedicated_storage_section()); + break; + case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplace: + fillConfigs(from.replace()); + break; + case Ydb::Config::ReplaceConfigRequest::ActionCase::ACTION_NOT_SET: + break; // TODO: handle as error? + } + + to->CopyFrom(NKikimr::NYaml::BuildInitDistributedStorageCommand(configStr)); return true; } @@ -105,9 +130,35 @@ class TReplaceStorageConfigRequest : public TBSConfigRequestGrpcSetYAML(GetProtoRequest()->main_config()); + + TString configStr; + + auto fillConfigs = [&](const auto& configBundle) { + for (const auto& config : configBundle.config()) { + if (NYamlConfig::IsMainConfig(config)) { + configStr = config; + } + } + }; + + switch (GetProtoRequest()->action_case()) { + case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceEnableDedicatedStorageSection: + fillConfigs(GetProtoRequest()->replace_enable_dedicated_storage_section()); + break; + case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceDisableDedicatedStorageSection: + fillConfigs(GetProtoRequest()->replace_disable_dedicated_storage_section()); + break; + case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceWithDedicatedStorageSection: + fillConfigs(GetProtoRequest()->replace_with_dedicated_storage_section()); + break; + case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplace: + fillConfigs(GetProtoRequest()->replace()); + break; + case Ydb::Config::ReplaceConfigRequest::ActionCase::ACTION_NOT_SET: + break; // TODO: handle as error? + } + + cmd->SetYAML(configStr); } void FillDistconfResult(NKikimrBlobStorage::TEvNodeConfigInvokeOnRootResult& /*record*/, @@ -153,14 +204,14 @@ class TReplaceStorageConfigRequest : public TBSConfigRequestGrpcreplace_enable_dedicated_storage_section()); + fillConfigs(request->replace_disable_dedicated_storage_section()); break; case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceWithDedicatedStorageSection: dedicated_config_mode = true; - fillConfigs(request->replace_enable_dedicated_storage_section()); - [[fallthrough]]; + fillConfigs(request->replace_with_dedicated_storage_section()); + break; case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplace: - fillConfigs(request->replace_enable_dedicated_storage_section()); + fillConfigs(request->replace()); break; case Ydb::Config::ReplaceConfigRequest::ActionCase::ACTION_NOT_SET: break; // TODO: handle as error? @@ -224,7 +275,7 @@ class TFetchStorageConfigRequest : public TBSConfigRequestGrpcGet()->Record; if (record.HasClusterYaml()) { - // FIXME: fill identity - result.add_config()->set_config(ev->Get()->Record.GetClusterYaml()); + auto conf = ev->Get()->Record.GetClusterYaml(); + auto& config = *result.add_config(); + auto& identity = *config.mutable_identity(); + auto metadata = NYamlConfig::GetMainMetadata(conf); + // TODO: !imp + identity.set_version(*metadata.Version); + identity.set_cluster(AppData()->ClusterName); + identity.mutable_main(); + config.set_config(conf); } if (record.HasStorageYaml()) { - // FIXME: fill identity - result.add_config()->set_config(ev->Get()->Record.GetStorageYaml()); + auto conf = ev->Get()->Record.GetStorageYaml(); + auto& config = *result.add_config(); + auto& identity = *config.mutable_identity(); + auto metadata = NYamlConfig::GetStorageMetadata(conf); + // TODO: !imp + identity.set_version(*metadata.Version); + identity.set_cluster(AppData()->ClusterName); + identity.mutable_storage(); + config.set_config(conf); } self->ReplyWithResult(Ydb::StatusIds::SUCCESS, result, self->ActorContext()); } else { diff --git a/ydb/library/yaml_config/public/yaml_config.cpp b/ydb/library/yaml_config/public/yaml_config.cpp index 3121a08638ab..dca48a8fd251 100644 --- a/ydb/library/yaml_config/public/yaml_config.cpp +++ b/ydb/library/yaml_config/public/yaml_config.cpp @@ -692,6 +692,19 @@ TDatabaseMetadata GetDatabaseMetadata(const TString& config) { return {}; } +TStorageMetadata GetStorageMetadata(const TString& config) { + if (auto doc = GetMetadataDoc(config); doc) { + auto versionNode = doc->Node["version"]; + auto clusterNode = doc->Node["cluster"]; + return TStorageMetadata{ + .Version = versionNode ? std::optional{FromString(versionNode.Scalar())} : std::nullopt, + .Cluster = clusterNode ? std::optional{clusterNode.Scalar()} : std::nullopt, + }; + } + + return {}; +} + TVolatileMetadata GetVolatileMetadata(const TString& config) { if (auto doc = GetMetadataDoc(config); doc) { auto versionNode = doc->Node.at("version"); diff --git a/ydb/library/yaml_config/public/yaml_config.h b/ydb/library/yaml_config/public/yaml_config.h index d541feeca274..090adbc93d4c 100644 --- a/ydb/library/yaml_config/public/yaml_config.h +++ b/ydb/library/yaml_config/public/yaml_config.h @@ -185,6 +185,14 @@ struct TMainMetadata { std::optional Cluster; }; +/** + * Represents config metadata + */ +struct TStorageMetadata { + std::optional Version; + std::optional Cluster; +}; + /** * Represents volatile config metadata */ @@ -222,6 +230,11 @@ TMainMetadata GetMainMetadata(const TString& config); */ TDatabaseMetadata GetDatabaseMetadata(const TString& config); +/** + * Parses storage config metadata + */ +TStorageMetadata GetStorageMetadata(const TString& config); + /** * Parses volatile config metadata */ From 85036cb317b1ebcb67ab6b549fb6e53fa777c341 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 16 Feb 2025 11:01:19 +0000 Subject: [PATCH 10/50] WIP --- ydb/core/grpc_services/rpc_bsconfig.cpp | 29 +++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/ydb/core/grpc_services/rpc_bsconfig.cpp b/ydb/core/grpc_services/rpc_bsconfig.cpp index 50aaf89062de..c4c0e6764521 100644 --- a/ydb/core/grpc_services/rpc_bsconfig.cpp +++ b/ydb/core/grpc_services/rpc_bsconfig.cpp @@ -168,8 +168,33 @@ class TReplaceStorageConfigRequest : public TBSConfigRequestGrpcmain_config()); + TString mainConfig; + + auto fillConfigs = [&](const auto& configBundle) { + for (const auto& config : configBundle.config()) { + if (NYamlConfig::IsMainConfig(config)) { + mainConfig = config; + } + } + }; + + switch (GetProtoRequest()->action_case()) { + case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceEnableDedicatedStorageSection: + fillConfigs(GetProtoRequest()->replace_enable_dedicated_storage_section()); + break; + case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceDisableDedicatedStorageSection: + fillConfigs(GetProtoRequest()->replace_disable_dedicated_storage_section()); + break; + case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceWithDedicatedStorageSection: + fillConfigs(GetProtoRequest()->replace_with_dedicated_storage_section()); + break; + case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplace: + fillConfigs(GetProtoRequest()->replace()); + break; + case Ydb::Config::ReplaceConfigRequest::ActionCase::ACTION_NOT_SET: + break; // TODO: handle as error? + } + newConfig = NYaml::Parse(mainConfig); // check allow unknown fields } catch (const std::exception&) { return false; // assuming no distconf enabled in this config } From 4c5bc2b3da67fbdffe3d0a0cb5ec9fddfeb411b2 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 16 Feb 2025 11:43:21 +0000 Subject: [PATCH 11/50] WIP --- ydb/core/grpc_services/rpc_bsconfig.cpp | 52 +++++-------- .../yaml_config/yaml_config_parser.cpp | 2 +- ydb/public/api/protos/ydb_bsconfig.proto | 73 ++++++++++--------- 3 files changed, 59 insertions(+), 68 deletions(-) diff --git a/ydb/core/grpc_services/rpc_bsconfig.cpp b/ydb/core/grpc_services/rpc_bsconfig.cpp index c4c0e6764521..c5cf3f416935 100644 --- a/ydb/core/grpc_services/rpc_bsconfig.cpp +++ b/ydb/core/grpc_services/rpc_bsconfig.cpp @@ -29,11 +29,7 @@ bool CopyToConfigRequest(const Ydb::Config::ReplaceConfigRequest &from, NKikimrB TString configStr; auto fillConfigs = [&](const auto& configBundle) { - for (const auto& config : configBundle.config()) { - if (NYamlConfig::IsMainConfig(config)) { - configStr = config; - } - } + configStr = configBundle.main_config(); }; switch (from.action_case()) { @@ -41,13 +37,13 @@ bool CopyToConfigRequest(const Ydb::Config::ReplaceConfigRequest &from, NKikimrB fillConfigs(from.replace_enable_dedicated_storage_section()); break; case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceDisableDedicatedStorageSection: - fillConfigs(from.replace_disable_dedicated_storage_section()); + configStr = from.replace_disable_dedicated_storage_section(); break; case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceWithDedicatedStorageSection: fillConfigs(from.replace_with_dedicated_storage_section()); break; case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplace: - fillConfigs(from.replace()); + configStr = from.replace(); break; case Ydb::Config::ReplaceConfigRequest::ActionCase::ACTION_NOT_SET: break; // TODO: handle as error? @@ -134,11 +130,7 @@ class TReplaceStorageConfigRequest : public TBSConfigRequestGrpcaction_case()) { @@ -146,13 +138,13 @@ class TReplaceStorageConfigRequest : public TBSConfigRequestGrpcreplace_enable_dedicated_storage_section()); break; case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceDisableDedicatedStorageSection: - fillConfigs(GetProtoRequest()->replace_disable_dedicated_storage_section()); + configStr = GetProtoRequest()->replace_disable_dedicated_storage_section(); break; case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceWithDedicatedStorageSection: fillConfigs(GetProtoRequest()->replace_with_dedicated_storage_section()); break; case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplace: - fillConfigs(GetProtoRequest()->replace()); + configStr = GetProtoRequest()->replace(); break; case Ydb::Config::ReplaceConfigRequest::ActionCase::ACTION_NOT_SET: break; // TODO: handle as error? @@ -168,14 +160,10 @@ class TReplaceStorageConfigRequest : public TBSConfigRequestGrpcaction_case()) { @@ -183,18 +171,18 @@ class TReplaceStorageConfigRequest : public TBSConfigRequestGrpcreplace_enable_dedicated_storage_section()); break; case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceDisableDedicatedStorageSection: - fillConfigs(GetProtoRequest()->replace_disable_dedicated_storage_section()); + configStr = GetProtoRequest()->replace_disable_dedicated_storage_section(); break; case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceWithDedicatedStorageSection: fillConfigs(GetProtoRequest()->replace_with_dedicated_storage_section()); break; case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplace: - fillConfigs(GetProtoRequest()->replace()); + configStr = GetProtoRequest()->replace(); break; case Ydb::Config::ReplaceConfigRequest::ActionCase::ACTION_NOT_SET: break; // TODO: handle as error? } - newConfig = NYaml::Parse(mainConfig); // check allow unknown fields + newConfig = NYaml::Parse(configStr); // check allow unknown fields } catch (const std::exception&) { return false; // assuming no distconf enabled in this config } @@ -210,14 +198,12 @@ class TReplaceStorageConfigRequest : public TBSConfigRequestGrpcreplace_disable_dedicated_storage_section()); + mainConfig = request->replace_disable_dedicated_storage_section(); break; case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceWithDedicatedStorageSection: dedicated_config_mode = true; fillConfigs(request->replace_with_dedicated_storage_section()); break; case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplace: - fillConfigs(request->replace()); + mainConfig = request->replace(); break; case Ydb::Config::ReplaceConfigRequest::ActionCase::ACTION_NOT_SET: break; // TODO: handle as error? @@ -294,7 +280,7 @@ class TFetchStorageConfigRequest : public TBSConfigRequestGrpcadd_config(data); + replaceRequest.set_replace(data); return replaceRequest; } diff --git a/ydb/public/api/protos/ydb_bsconfig.proto b/ydb/public/api/protos/ydb_bsconfig.proto index 759673a2d68c..063333ff2022 100644 --- a/ydb/public/api/protos/ydb_bsconfig.proto +++ b/ydb/public/api/protos/ydb_bsconfig.proto @@ -40,28 +40,27 @@ message ConfigIdentity { message ReplaceConfigRequest { Ydb.Operations.OperationParams operation_params = 1; - // Zero or more configs of any type. - // Exact type of config identified by metadata stored inside config. message ConfigBundle { - repeated string config = 1; + optional string main_config = 1; + optional string storage_config = 2; } oneof action { - // Replace config on cluster. - // By default awaits single config with matching metadata (kind MainConfig, right cluster and version + 1) - ConfigBundle replace = 2; + // Replaces config on cluster. + // By default awaits config with matching metadata (kind MainConfig, right cluster and version + 1) + string replace = 2; - // Replace storage and/or main configs on cluster in dedicated storage config mode + // Replaces storage and/or main configs on cluster in dedicated storage config mode // By default awaits either MainConfig or StorageConfig or both with matching metadata ConfigBundle replace_with_dedicated_storage_section = 3; - // Replace storage and main configs on cluster in single config mode **AND** enables dedicated storage config mode + // Replaces storage and main configs on cluster in single config mode **AND** enables dedicated storage config mode // By default awaits MainConfig and StorageConfig with matching metadata ConfigBundle replace_enable_dedicated_storage_section = 4; - // Replace main config on cluster in single config mode **AND** disables dedicated storage config mode + // Replaces main config on cluster in single config mode **AND** disables dedicated storage config mode // By default awaits MainConfig with matching metadata - ConfigBundle replace_disable_dedicated_storage_section = 5; + string replace_disable_dedicated_storage_section = 5; } // Try to apply config(s) without making real changes @@ -70,7 +69,7 @@ message ReplaceConfigRequest { // All errors returned as issues in response bool dry_run = 6; - // Allows to apply config with fields unknown to out parser + // Allows to apply config with fields unknown to config parser // It is useful when user need to try some new experimental flag, then rollback to previous version // and want to update existing config keeping flag from newer version in config // All unknown fields still reported in result as warnings @@ -92,32 +91,38 @@ message ReplaceConfigResponse { message ReplaceConfigResult {} -message FetchModeAll { - // Use this option to somehow transform main config - // Currently used to automatically derive detached StorageSection from MainConfig - // or vice versa - oneof config_transform { - // Optionally may be set to explicitly tell "do not transform anything" - google.protobuf.Empty none = 1; - - // Fetch will return MainConfig and StorageConfig derived from MainConfig - // by detaching storage config - google.protobuf.Empty detach_storage_config_section = 2; - - // Fetch will return MainConfig derived from MainConfig and StorageConfig - // by attaching storage config to main config - google.protobuf.Empty attach_storage_config_section = 3; - } -} - -message FetchModeByIdentity { - // List of configs to fetch - repeated ConfigIdentity identity = 1; -} - message FetchConfigRequest { Ydb.Operations.OperationParams operation_params = 1; + // Settings for mode "all" + // In this mode server will return all configs to user + // There are additional settings allowing to transform configs in server answer + message FetchModeAll { + // Use this option to somehow transform main config + // Currently used to automatically derive detached StorageSection from MainConfig + // or vice versa + oneof config_transform { + // Optionally may be set to explicitly tell "do not transform anything" + google.protobuf.Empty none = 1; + + // Fetch will return MainConfig and StorageConfig derived from MainConfig + // by detaching storage config + google.protobuf.Empty detach_storage_config_section = 2; + + // Fetch will return MainConfig derived from MainConfig and StorageConfig + // by attaching storage config to main config + google.protobuf.Empty attach_storage_config_section = 3; + } + } + + // Settings for mode "by identity" + // In this mode server will return all configs with identites which are exactly match to identites in request + // If there is no matching identity on server the identity will be skipped + message FetchModeByIdentity { + // List of configs to fetch + repeated ConfigIdentity identity = 1; + } + // Depending on mode Fetch will return different set of configs // List of modes may be extended in future oneof mode { From 5ee283c4a5abb7d3338d06a6df432f3a5ed8adf6 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 16 Feb 2025 11:48:42 +0000 Subject: [PATCH 12/50] WIP --- ydb/public/api/grpc/ya.make | 6 +++--- ...ydb_bsconfig_v1.proto => ydb_config_v1.proto} | 0 ydb/public/api/protos/ya.make | 16 ++++++++-------- .../{ydb_bsconfig.proto => ydb_config.proto} | 0 4 files changed, 11 insertions(+), 11 deletions(-) rename ydb/public/api/grpc/{ydb_bsconfig_v1.proto => ydb_config_v1.proto} (100%) rename ydb/public/api/protos/{ydb_bsconfig.proto => ydb_config.proto} (100%) diff --git a/ydb/public/api/grpc/ya.make b/ydb/public/api/grpc/ya.make index cfd27e416671..811a5ebe927f 100644 --- a/ydb/public/api/grpc/ya.make +++ b/ydb/public/api/grpc/ya.make @@ -5,15 +5,16 @@ MAVEN_GROUP_ID(com.yandex.ydb) GRPC() SRCS( - ydb_federation_discovery_v1.proto - ydb_bsconfig_v1.proto ydb_auth_v1.proto ydb_cms_v1.proto + ydb_config_v1.proto ydb_coordination_v1.proto ydb_debug_v1.proto ydb_discovery_v1.proto ydb_export_v1.proto + ydb_federation_discovery_v1.proto ydb_import_v1.proto + ydb_keyvalue_v1.proto ydb_monitoring_v1.proto ydb_operation_v1.proto ydb_query_v1.proto @@ -22,7 +23,6 @@ SRCS( ydb_scripting_v1.proto ydb_table_v1.proto ydb_topic_v1.proto - ydb_keyvalue_v1.proto ) PEERDIR( diff --git a/ydb/public/api/grpc/ydb_bsconfig_v1.proto b/ydb/public/api/grpc/ydb_config_v1.proto similarity index 100% rename from ydb/public/api/grpc/ydb_bsconfig_v1.proto rename to ydb/public/api/grpc/ydb_config_v1.proto diff --git a/ydb/public/api/protos/ya.make b/ydb/public/api/protos/ya.make index f6aebfb41e7b..5565a61892eb 100644 --- a/ydb/public/api/protos/ya.make +++ b/ydb/public/api/protos/ya.make @@ -8,6 +8,7 @@ PEERDIR( SRCS( draft/datastreams.proto + draft/field_transformation.proto draft/fq.proto draft/persqueue_common.proto draft/persqueue_error_codes.proto @@ -20,26 +21,27 @@ SRCS( draft/ydb_tablet.proto draft/ydb_view.proto draft/ymq.proto - draft/field_transformation.proto - ydb_federation_discovery.proto persqueue_error_codes_v1.proto ydb_auth.proto - ydb_debug.proto - ydb_persqueue_v1.proto - ydb_persqueue_cluster_discovery.proto ydb_clickhouse_internal.proto ydb_cms.proto ydb_common.proto + ydb_config.proto ydb_coordination.proto + ydb_debug.proto ydb_discovery.proto ydb_export.proto + ydb_federation_discovery.proto ydb_formats.proto ydb_import.proto ydb_issue_message.proto + ydb_keyvalue.proto ydb_monitoring.proto ydb_operation.proto - ydb_query_stats.proto + ydb_persqueue_cluster_discovery.proto + ydb_persqueue_v1.proto ydb_query.proto + ydb_query_stats.proto ydb_rate_limiter.proto ydb_scheme.proto ydb_scripting.proto @@ -47,8 +49,6 @@ SRCS( ydb_table.proto ydb_topic.proto ydb_value.proto - ydb_keyvalue.proto - ydb_bsconfig.proto ) CPP_PROTO_PLUGIN0(validation ydb/public/lib/validation) diff --git a/ydb/public/api/protos/ydb_bsconfig.proto b/ydb/public/api/protos/ydb_config.proto similarity index 100% rename from ydb/public/api/protos/ydb_bsconfig.proto rename to ydb/public/api/protos/ydb_config.proto From f35b32626fa7a041255810bccca5b9e20162fd37 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 16 Feb 2025 11:58:24 +0000 Subject: [PATCH 13/50] WIP --- ydb/core/grpc_services/rpc_bsconfig_base.h | 2 +- ydb/library/yaml_config/yaml_config_parser.h | 2 +- ydb/public/api/grpc/ydb_config_v1.proto | 2 +- ydb/services/bsconfig/grpc_service.h | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ydb/core/grpc_services/rpc_bsconfig_base.h b/ydb/core/grpc_services/rpc_bsconfig_base.h index 5895ca52aa03..e78c586c87ab 100644 --- a/ydb/core/grpc_services/rpc_bsconfig_base.h +++ b/ydb/core/grpc_services/rpc_bsconfig_base.h @@ -3,7 +3,7 @@ #include "rpc_deferrable.h" -#include +#include #include #include #include diff --git a/ydb/library/yaml_config/yaml_config_parser.h b/ydb/library/yaml_config/yaml_config_parser.h index 7e1928f4c96e..4739ee3ac945 100644 --- a/ydb/library/yaml_config/yaml_config_parser.h +++ b/ydb/library/yaml_config/yaml_config_parser.h @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include diff --git a/ydb/public/api/grpc/ydb_config_v1.proto b/ydb/public/api/grpc/ydb_config_v1.proto index d8861ae9d0e9..842522183233 100644 --- a/ydb/public/api/grpc/ydb_config_v1.proto +++ b/ydb/public/api/grpc/ydb_config_v1.proto @@ -6,7 +6,7 @@ option java_package = "com.yandex.ydb.bsconfig.v1"; option java_outer_classname = "ConfigGrpc"; option java_multiple_files = true; -import "ydb/public/api/protos/ydb_bsconfig.proto"; +import "ydb/public/api/protos/ydb_config.proto"; service ConfigService { diff --git a/ydb/services/bsconfig/grpc_service.h b/ydb/services/bsconfig/grpc_service.h index 18d95b2071e0..a83424748451 100644 --- a/ydb/services/bsconfig/grpc_service.h +++ b/ydb/services/bsconfig/grpc_service.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include From 8f484613f739f4f96796937ed4e70e81ff7aebda Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 16 Feb 2025 12:01:43 +0000 Subject: [PATCH 14/50] WIP --- .../{rpc_bsconfig.cpp => rpc_config.cpp} | 0 ...{rpc_bsconfig_base.h => rpc_config_base.h} | 0 ydb/core/grpc_services/ya.make | 24 +++++++++---------- 3 files changed, 12 insertions(+), 12 deletions(-) rename ydb/core/grpc_services/{rpc_bsconfig.cpp => rpc_config.cpp} (100%) rename ydb/core/grpc_services/{rpc_bsconfig_base.h => rpc_config_base.h} (100%) diff --git a/ydb/core/grpc_services/rpc_bsconfig.cpp b/ydb/core/grpc_services/rpc_config.cpp similarity index 100% rename from ydb/core/grpc_services/rpc_bsconfig.cpp rename to ydb/core/grpc_services/rpc_config.cpp diff --git a/ydb/core/grpc_services/rpc_bsconfig_base.h b/ydb/core/grpc_services/rpc_config_base.h similarity index 100% rename from ydb/core/grpc_services/rpc_bsconfig_base.h rename to ydb/core/grpc_services/rpc_config_base.h diff --git a/ydb/core/grpc_services/ya.make b/ydb/core/grpc_services/ya.make index bd7d0a737703..a68043bfd308 100644 --- a/ydb/core/grpc_services/ya.make +++ b/ydb/core/grpc_services/ya.make @@ -5,8 +5,8 @@ ADDINCL( ) SRCS( - audit_log.cpp audit_dml_operations.cpp + audit_log.cpp audit_logins.cpp db_metadata_cache.h grpc_endpoint_publish_actor.cpp @@ -22,32 +22,32 @@ SRCS( rpc_alter_table.cpp rpc_backup.cpp rpc_begin_transaction.cpp - rpc_bsconfig.cpp rpc_calls.cpp rpc_cancel_operation.cpp rpc_cms.cpp rpc_commit_transaction.cpp - rpc_dynamic_config.cpp + rpc_config.cpp rpc_copy_table.cpp rpc_copy_tables.cpp - rpc_export.cpp rpc_create_coordination_node.cpp rpc_create_table.cpp rpc_describe_coordination_node.cpp rpc_describe_path.cpp rpc_describe_table.cpp rpc_describe_table_options.cpp + rpc_discovery.cpp rpc_drop_coordination_node.cpp rpc_drop_table.cpp - rpc_discovery.cpp + rpc_dynamic_config.cpp rpc_execute_data_query.cpp rpc_execute_scheme_query.cpp rpc_execute_yql_script.cpp - rpc_explain_yql_script.cpp rpc_explain_data_query.cpp + rpc_explain_yql_script.cpp + rpc_export.cpp rpc_forget_operation.cpp - rpc_fq_internal.cpp rpc_fq.cpp + rpc_fq_internal.cpp rpc_get_operation.cpp rpc_get_scale_recommendation.cpp rpc_get_shard_locations.cpp @@ -59,20 +59,21 @@ SRCS( rpc_kh_snapshots.cpp rpc_kqp_base.cpp rpc_list_operations.cpp - rpc_login.cpp rpc_load_rows.cpp rpc_log_store.cpp - rpc_node_registration.cpp + rpc_login.cpp rpc_maintenance.cpp rpc_make_directory.cpp rpc_modify_permissions.cpp rpc_monitoring.cpp + rpc_node_registration.cpp + rpc_object_storage.cpp rpc_ping.cpp rpc_prepare_data_query.cpp rpc_rate_limiter_api.cpp rpc_read_columns.cpp - rpc_read_table.cpp rpc_read_rows.cpp + rpc_read_table.cpp rpc_remove_directory.cpp rpc_rename_tables.cpp rpc_replication.cpp @@ -80,9 +81,8 @@ SRCS( rpc_scheme_base.cpp rpc_stream_execute_scan_query.cpp rpc_stream_execute_yql_script.cpp - rpc_whoami.cpp - rpc_object_storage.cpp rpc_view.cpp + rpc_whoami.cpp table_settings.cpp rpc_common/rpc_common_kqp_session.cpp From 84a553d42c0365de701a95647a781ba49fef61a0 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 16 Feb 2025 12:04:16 +0000 Subject: [PATCH 15/50] WIP --- ydb/core/grpc_services/rpc_config.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/core/grpc_services/rpc_config.cpp b/ydb/core/grpc_services/rpc_config.cpp index c5cf3f416935..ff2c81b36a46 100644 --- a/ydb/core/grpc_services/rpc_config.cpp +++ b/ydb/core/grpc_services/rpc_config.cpp @@ -2,7 +2,7 @@ #include #include #include -#include "rpc_bsconfig_base.h" +#include "rpc_config_base.h" #include #include From b77934834e8f5e5f38d29552cc2854d68118cff7 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 16 Feb 2025 12:07:11 +0000 Subject: [PATCH 16/50] WIP --- ydb/public/api/grpc/ydb_config_v1.proto | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/ydb/public/api/grpc/ydb_config_v1.proto b/ydb/public/api/grpc/ydb_config_v1.proto index 842522183233..2d6e9c503fb0 100644 --- a/ydb/public/api/grpc/ydb_config_v1.proto +++ b/ydb/public/api/grpc/ydb_config_v1.proto @@ -2,21 +2,19 @@ syntax = "proto3"; package Ydb.Config.V1; -option java_package = "com.yandex.ydb.bsconfig.v1"; +option java_package = "com.yandex.ydb.config.v1"; option java_outer_classname = "ConfigGrpc"; option java_multiple_files = true; import "ydb/public/api/protos/ydb_config.proto"; service ConfigService { - - // Initialize Blobstorage/single config + // Replace one or many configs. rpc ReplaceConfig(Config.ReplaceConfigRequest) returns (Config.ReplaceConfigResponse); - // Fetch Blobstorage/single config + // Fetch configs from cluster. rpc FetchConfig(Config.FetchConfigRequest) returns (Config.FetchConfigResponse); - // Bootstrap automatically configured cluster + // Start cluster initialization process. rpc BootstrapCluster(Config.BootstrapClusterRequest) returns (Config.BootstrapClusterResponse); - } From 5fe9f691febe64e93147fb2f36fcbbf5fb032705 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 16 Feb 2025 12:23:06 +0000 Subject: [PATCH 17/50] WIP --- ydb/public/lib/ydb_cli/commands/ya.make | 2 +- ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp | 2 +- ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp | 2 +- ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp | 2 +- .../sdk/cpp/client/{ydb_bsconfig => ydb_config}/ya.make | 2 +- .../ydb_storage_config.cpp => ydb_config/ydb_config.cpp} | 0 .../ydb_storage_config.h => ydb_config/ydb_config.h} | 2 +- .../client/{bsconfig/storage_config.h => config/config.h} | 0 .../client/{bsconfig/storage_config.cpp => config/config.cpp} | 4 ++-- ydb/public/sdk/cpp/src/client/{bsconfig => config}/ya.make | 2 +- 10 files changed, 9 insertions(+), 9 deletions(-) rename ydb/public/sdk/cpp/client/{ydb_bsconfig => ydb_config}/ya.make (87%) rename ydb/public/sdk/cpp/client/{ydb_bsconfig/ydb_storage_config.cpp => ydb_config/ydb_config.cpp} (100%) rename ydb/public/sdk/cpp/client/{ydb_bsconfig/ydb_storage_config.h => ydb_config/ydb_config.h} (96%) rename ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/{bsconfig/storage_config.h => config/config.h} (100%) rename ydb/public/sdk/cpp/src/client/{bsconfig/storage_config.cpp => config/config.cpp} (97%) rename ydb/public/sdk/cpp/src/client/{bsconfig => config}/ya.make (91%) diff --git a/ydb/public/lib/ydb_cli/commands/ya.make b/ydb/public/lib/ydb_cli/commands/ya.make index 556f632b32bd..7ded23503157 100644 --- a/ydb/public/lib/ydb_cli/commands/ya.make +++ b/ydb/public/lib/ydb_cli/commands/ya.make @@ -59,7 +59,7 @@ PEERDIR( ydb/public/lib/ydb_cli/dump/files ydb/public/lib/ydb_cli/import ydb/public/lib/ydb_cli/topic - ydb/public/sdk/cpp/src/client/bsconfig + ydb/public/sdk/cpp/src/client/config ydb/public/sdk/cpp/src/client/coordination ydb/public/sdk/cpp/src/client/debug ydb/public/sdk/cpp/src/client/draft diff --git a/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp b/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp index 959873ab883f..e0e6a8c48399 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp @@ -2,7 +2,7 @@ #include "ydb_dynamic_config.h" -#include +#include #include #define INCLUDE_YDB_INTERNAL_H diff --git a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp index 1c13650a3e02..1c113913a424 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp @@ -1,7 +1,7 @@ #include "ydb_dynamic_config.h" #include -#include +#include #include #include diff --git a/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp b/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp index 057a5ed7b3c2..166bb3c2341c 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp @@ -1,6 +1,6 @@ #include "ydb_storage_config.h" -#include +#include #include #include diff --git a/ydb/public/sdk/cpp/client/ydb_bsconfig/ya.make b/ydb/public/sdk/cpp/client/ydb_config/ya.make similarity index 87% rename from ydb/public/sdk/cpp/client/ydb_bsconfig/ya.make rename to ydb/public/sdk/cpp/client/ydb_config/ya.make index 28a8afbabafe..578691b9d383 100644 --- a/ydb/public/sdk/cpp/client/ydb_bsconfig/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_config/ya.make @@ -1,7 +1,7 @@ LIBRARY() SRCS( - ydb_storage_config.cpp + ydb_config.cpp ) PEERDIR( diff --git a/ydb/public/sdk/cpp/client/ydb_bsconfig/ydb_storage_config.cpp b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp similarity index 100% rename from ydb/public/sdk/cpp/client/ydb_bsconfig/ydb_storage_config.cpp rename to ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp diff --git a/ydb/public/sdk/cpp/client/ydb_bsconfig/ydb_storage_config.h b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h similarity index 96% rename from ydb/public/sdk/cpp/client/ydb_bsconfig/ydb_storage_config.h rename to ydb/public/sdk/cpp/client/ydb_config/ydb_config.h index a2178fc29ccc..59a7b36d684a 100644 --- a/ydb/public/sdk/cpp/client/ydb_bsconfig/ydb_storage_config.h +++ b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include #include diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/bsconfig/storage_config.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/config/config.h similarity index 100% rename from ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/bsconfig/storage_config.h rename to ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/config/config.h diff --git a/ydb/public/sdk/cpp/src/client/bsconfig/storage_config.cpp b/ydb/public/sdk/cpp/src/client/config/config.cpp similarity index 97% rename from ydb/public/sdk/cpp/src/client/bsconfig/storage_config.cpp rename to ydb/public/sdk/cpp/src/client/config/config.cpp index 6b25657e400e..eb291f2f2a6d 100644 --- a/ydb/public/sdk/cpp/src/client/bsconfig/storage_config.cpp +++ b/ydb/public/sdk/cpp/src/client/config/config.cpp @@ -1,9 +1,9 @@ -#include +#include #include #include -#include +#include namespace NYdb::inline V3::NStorageConfig { diff --git a/ydb/public/sdk/cpp/src/client/bsconfig/ya.make b/ydb/public/sdk/cpp/src/client/config/ya.make similarity index 91% rename from ydb/public/sdk/cpp/src/client/bsconfig/ya.make rename to ydb/public/sdk/cpp/src/client/config/ya.make index c329c69b0e18..d129149f9fb6 100644 --- a/ydb/public/sdk/cpp/src/client/bsconfig/ya.make +++ b/ydb/public/sdk/cpp/src/client/config/ya.make @@ -3,7 +3,7 @@ LIBRARY() INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) SRCS( - storage_config.cpp + config.cpp ) PEERDIR( From 917088587c155f5b6a2602b5bddd84a87460e71e Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 16 Feb 2025 12:48:25 +0000 Subject: [PATCH 18/50] WIP --- ydb/core/driver_lib/run/run.cpp | 2 +- ydb/core/driver_lib/run/ya.make | 50 +++++++++---------- .../{bsconfig => config}/bsconfig_ut.cpp | 0 .../{bsconfig => config}/grpc_service.cpp | 0 .../{bsconfig => config}/grpc_service.h | 0 ydb/services/{bsconfig => config}/ut/ya.make | 0 ydb/services/{bsconfig => config}/ya.make | 0 ydb/services/ya.make | 2 +- 8 files changed, 27 insertions(+), 27 deletions(-) rename ydb/services/{bsconfig => config}/bsconfig_ut.cpp (100%) rename ydb/services/{bsconfig => config}/grpc_service.cpp (100%) rename ydb/services/{bsconfig => config}/grpc_service.h (100%) rename ydb/services/{bsconfig => config}/ut/ya.make (100%) rename ydb/services/{bsconfig => config}/ya.make (100%) diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index 5835a8e5a861..96e4a9d9ee28 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -99,7 +99,7 @@ #include #include -#include +#include #include #include #include diff --git a/ydb/core/driver_lib/run/ya.make b/ydb/core/driver_lib/run/ya.make index 34d6553c3486..3f72b78c5cfa 100644 --- a/ydb/core/driver_lib/run/ya.make +++ b/ydb/core/driver_lib/run/ya.make @@ -36,17 +36,7 @@ SRCS( PEERDIR( contrib/libs/protobuf - ydb/library/actors/core - ydb/library/actors/dnsresolver - ydb/library/actors/interconnect - ydb/library/actors/memory_log - ydb/library/actors/prof - ydb/library/actors/protos - ydb/library/actors/util library/cpp/getopt/small - ydb/public/sdk/cpp/src/library/grpc/client - ydb/library/grpc/server - ydb/library/grpc/server/actors library/cpp/logger library/cpp/malloc/api library/cpp/messagebus @@ -57,8 +47,8 @@ PEERDIR( library/cpp/svnversion ydb/core/actorlib_impl ydb/core/audit - ydb/core/base ydb/core/backup/controller + ydb/core/base ydb/core/blob_depot ydb/core/blobstorage ydb/core/blobstorage/backpressure @@ -86,10 +76,10 @@ PEERDIR( ydb/core/health_check ydb/core/http_proxy ydb/core/jaeger_tracing + ydb/core/kafka_proxy ydb/core/kesus/proxy ydb/core/kesus/tablet ydb/core/keyvalue - ydb/core/kafka_proxy ydb/core/kqp ydb/core/kqp/federated_query ydb/core/kqp/finalize_script_service @@ -124,11 +114,11 @@ PEERDIR( ydb/core/tracing ydb/core/tx ydb/core/tx/columnshard - ydb/core/tx/coordinator ydb/core/tx/conveyor/service - ydb/core/tx/limiter/service - ydb/core/tx/limiter/grouped_memory/usage + ydb/core/tx/coordinator ydb/core/tx/datashard + ydb/core/tx/limiter/grouped_memory/usage + ydb/core/tx/limiter/service ydb/core/tx/long_tx_service ydb/core/tx/long_tx_service/public ydb/core/tx/mediator @@ -145,43 +135,53 @@ PEERDIR( ydb/core/viewer ydb/core/ymq/actor ydb/core/ymq/http + ydb/library/actors/core + ydb/library/actors/dnsresolver + ydb/library/actors/interconnect + ydb/library/actors/memory_log + ydb/library/actors/prof + ydb/library/actors/protos + ydb/library/actors/util ydb/library/folder_service ydb/library/folder_service/proto + ydb/library/grpc/server + ydb/library/grpc/server/actors ydb/library/pdisk_io ydb/library/security - yql/essentials/minikql/comp_nodes/llvm14 - yt/yql/providers/yt/codec/codegen - yt/yql/providers/yt/comp_nodes/llvm14 ydb/library/yql/providers/pq/cm_client ydb/library/yql/providers/s3/actors - yql/essentials/public/udf/service/exception_policy ydb/public/lib/base ydb/public/lib/deprecated/client + ydb/public/sdk/cpp/src/library/grpc/client ydb/services/auth ydb/services/backup - ydb/services/bsconfig ydb/services/cms - ydb/services/dynamic_config + ydb/services/config ydb/services/datastreams + ydb/services/deprecated/persqueue_v0 ydb/services/discovery + ydb/services/dynamic_config + ydb/services/ext_index/metadata + ydb/services/ext_index/service ydb/services/fq ydb/services/kesus ydb/services/keyvalue ydb/services/local_discovery ydb/services/maintenance - ydb/services/metadata/ds_table ydb/services/metadata - ydb/services/ext_index/service - ydb/services/ext_index/metadata + ydb/services/metadata/ds_table ydb/services/monitoring ydb/services/persqueue_cluster_discovery - ydb/services/deprecated/persqueue_v0 ydb/services/persqueue_v1 ydb/services/rate_limiter ydb/services/replication ydb/services/tablet ydb/services/view ydb/services/ydb + yql/essentials/minikql/comp_nodes/llvm14 + yql/essentials/public/udf/service/exception_policy + yt/yql/providers/yt/codec/codegen + yt/yql/providers/yt/comp_nodes/llvm14 ) YQL_LAST_ABI_VERSION() diff --git a/ydb/services/bsconfig/bsconfig_ut.cpp b/ydb/services/config/bsconfig_ut.cpp similarity index 100% rename from ydb/services/bsconfig/bsconfig_ut.cpp rename to ydb/services/config/bsconfig_ut.cpp diff --git a/ydb/services/bsconfig/grpc_service.cpp b/ydb/services/config/grpc_service.cpp similarity index 100% rename from ydb/services/bsconfig/grpc_service.cpp rename to ydb/services/config/grpc_service.cpp diff --git a/ydb/services/bsconfig/grpc_service.h b/ydb/services/config/grpc_service.h similarity index 100% rename from ydb/services/bsconfig/grpc_service.h rename to ydb/services/config/grpc_service.h diff --git a/ydb/services/bsconfig/ut/ya.make b/ydb/services/config/ut/ya.make similarity index 100% rename from ydb/services/bsconfig/ut/ya.make rename to ydb/services/config/ut/ya.make diff --git a/ydb/services/bsconfig/ya.make b/ydb/services/config/ya.make similarity index 100% rename from ydb/services/bsconfig/ya.make rename to ydb/services/config/ya.make diff --git a/ydb/services/ya.make b/ydb/services/ya.make index cd73f2e86e6d..aa339d9de37b 100644 --- a/ydb/services/ya.make +++ b/ydb/services/ya.make @@ -2,8 +2,8 @@ RECURSE( auth backup bg_tasks - bsconfig cms + config datastreams deprecated/persqueue_v0 discovery From 8231e77528eb26552cbda8fb07e978731f250ad7 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 16 Feb 2025 13:07:48 +0000 Subject: [PATCH 19/50] WIP --- .../sdk/cpp/client/ydb_config/ydb_config.cpp | 42 +++++++++---------- .../sdk/cpp/client/ydb_config/ydb_config.h | 20 ++++----- .../ydb-cpp-sdk/client/config/config.h | 16 +++---- .../sdk/cpp/src/client/config/config.cpp | 16 +++---- 4 files changed, 47 insertions(+), 47 deletions(-) diff --git a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp index bf03e9a5ff00..63a95c1a324c 100644 --- a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp +++ b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp @@ -3,42 +3,42 @@ #include #include -namespace NYdb::NStorageConfig { +namespace NYdb::NConfig { -class TStorageConfigClient::TImpl : public TClientImplCommon { +class TConfigClient::TImpl : public TClientImplCommon { public: TImpl(std::shared_ptr connections, const TCommonClientSettings& settings) : TClientImplCommon(std::move(connections), settings) { } - TAsyncStatus ReplaceStorageConfig(const TString& config) { - auto request = MakeRequest(); + TAsyncStatus ReplaceConfig(const TString& config) { + auto request = MakeRequest(); request.set_yaml_config(config); - return RunSimple( + return RunSimple( std::move(request), - &Ydb::Config::V1::BSConfigService::Stub::AsyncReplaceStorageConfig); + &Ydb::Config::V1::BSConfigService::Stub::AsyncReplaceConfig); } - TAsyncFetchStorageConfigResult FetchStorageConfig(const TStorageConfigSettings& settings = {}) { - auto request = MakeOperationRequest(settings); - auto promise = NThreading::NewPromise(); + TAsyncFetchConfigResult FetchConfig(const TConfigSettings& settings = {}) { + auto request = MakeOperationRequest(settings); + auto promise = NThreading::NewPromise(); auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { TString config; - if (Ydb::Config::FetchStorageConfigResult result; any && any->UnpackTo(&result)) { + if (Ydb::Config::FetchConfigResult result; any && any->UnpackTo(&result)) { config = result.yaml_config(); } - TFetchStorageConfigResult val(TStatus(std::move(status)), std::move(config)); + TFetchConfigResult val(TStatus(std::move(status)), std::move(config)); promise.SetValue(std::move(val)); }; - Connections_->RunDeferred( + Connections_->RunDeferred( std::move(request), extractor, - &Ydb::Config::V1::BSConfigService::Stub::AsyncFetchStorageConfig, + &Ydb::Config::V1::BSConfigService::Stub::AsyncFetchConfig, DbDriverState_, INITIAL_DEFERRED_CALL_DELAY, TRpcRequestSettings::Make(settings)); @@ -55,21 +55,21 @@ class TStorageConfigClient::TImpl : public TClientImplCommonReplaceStorageConfig(config); +TAsyncStatus TConfigClient::ReplaceConfig(const TString& config) { + return Impl_->ReplaceConfig(config); } -TAsyncFetchStorageConfigResult TStorageConfigClient::FetchStorageConfig(const TStorageConfigSettings& settings) { - return Impl_->FetchStorageConfig(settings); +TAsyncFetchConfigResult TConfigClient::FetchConfig(const TConfigSettings& settings) { + return Impl_->FetchConfig(settings); } -TAsyncStatus TStorageConfigClient::BootstrapCluster(const TString& selfAssemblyUUID) { +TAsyncStatus TConfigClient::BootstrapCluster(const TString& selfAssemblyUUID) { return Impl_->BootstrapCluster(selfAssemblyUUID); } diff --git a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h index 59a7b36d684a..3eacad66d1c3 100644 --- a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h +++ b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h @@ -11,10 +11,10 @@ #include -namespace NYdb::NStorageConfig { +namespace NYdb::NConfig { -struct TFetchStorageConfigResult : public TStatus { - TFetchStorageConfigResult( +struct TFetchConfigResult : public TStatus { + TFetchConfigResult( TStatus&& status, TString&& config) : TStatus(std::move(status)) @@ -29,22 +29,22 @@ struct TFetchStorageConfigResult : public TStatus { TString Config_; }; -using TAsyncFetchStorageConfigResult = NThreading::TFuture; +using TAsyncFetchConfigResult = NThreading::TFuture; -struct TStorageConfigSettings : public NYdb::TOperationRequestSettings {}; +struct TConfigSettings : public NYdb::TOperationRequestSettings {}; -class TStorageConfigClient { +class TConfigClient { public: - explicit TStorageConfigClient(const TDriver& driver, const TCommonClientSettings& settings = {}); + explicit TConfigClient(const TDriver& driver, const TCommonClientSettings& settings = {}); - ~TStorageConfigClient(); + ~TConfigClient(); // Replace config - TAsyncStatus ReplaceStorageConfig(const TString& config); + TAsyncStatus ReplaceConfig(const TString& config); // Fetch current cluster storage config - TAsyncFetchStorageConfigResult FetchStorageConfig(const TStorageConfigSettings& settings = {}); + TAsyncFetchConfigResult FetchConfig(const TConfigSettings& settings = {}); // Bootstrap cluster with automatic configuration TAsyncStatus BootstrapCluster(const TString& selfAssemblyUUID); diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/config/config.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/config/config.h index 33f56c28bf72..a2f06d37e7e3 100644 --- a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/config/config.h +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/config/config.h @@ -9,7 +9,7 @@ #include #include -namespace NYdb::inline V3::NStorageConfig { +namespace NYdb::inline V3::NConfig { struct TFetchConfigResult : public TStatus { TFetchConfigResult( @@ -18,20 +18,20 @@ struct TFetchConfigResult : public TStatus { std::string&& storage_config) : TStatus(std::move(status)) , Config_(std::move(config)) - , StorageConfig_(std::move(storage_config)) + , Config_(std::move(storage_config)) {} const std::string& GetConfig() const { return Config_; } - const std::string& GetStorageConfig() const { - return StorageConfig_; + const std::string& GetConfig() const { + return Config_; } private: std::string Config_; - std::string StorageConfig_; + std::string Config_; }; using TAsyncFetchConfigResult = NThreading::TFuture; @@ -50,11 +50,11 @@ struct TFetchConfigSettings : public NYdb::TOperationRequestSettings {}; -class TStorageConfigClient { +class TConfigClient { public: - explicit TStorageConfigClient(const TDriver& driver, const TCommonClientSettings& settings = {}); + explicit TConfigClient(const TDriver& driver, const TCommonClientSettings& settings = {}); - ~TStorageConfigClient(); + ~TConfigClient(); // Replace config TAsyncStatus ReplaceConfig( diff --git a/ydb/public/sdk/cpp/src/client/config/config.cpp b/ydb/public/sdk/cpp/src/client/config/config.cpp index eb291f2f2a6d..cc960bff550b 100644 --- a/ydb/public/sdk/cpp/src/client/config/config.cpp +++ b/ydb/public/sdk/cpp/src/client/config/config.cpp @@ -5,9 +5,9 @@ #include -namespace NYdb::inline V3::NStorageConfig { +namespace NYdb::inline V3::NConfig { -class TStorageConfigClient::TImpl : public TClientImplCommon { +class TConfigClient::TImpl : public TClientImplCommon { public: TImpl(std::shared_ptr connections, const TCommonClientSettings& settings) : TClientImplCommon(std::move(connections), settings) @@ -91,13 +91,13 @@ class TStorageConfigClient::TImpl : public TClientImplCommon& yaml_config, const std::optional& storage_yaml_config, const TReplaceConfigSettings& settings) @@ -105,13 +105,13 @@ TAsyncStatus TStorageConfigClient::ReplaceConfig( return Impl_->ReplaceConfig(yaml_config, storage_yaml_config, settings); } -TAsyncFetchConfigResult TStorageConfigClient::FetchConfig(bool dedicated_storage_section, +TAsyncFetchConfigResult TConfigClient::FetchConfig(bool dedicated_storage_section, bool dedicated_cluster_section, const TFetchConfigSettings& settings) { return Impl_->FetchConfig(dedicated_storage_section, dedicated_cluster_section, settings); } -TAsyncStatus TStorageConfigClient::BootstrapCluster(const std::string& selfAssemblyUUID, const TBootstrapClusterSettings& settings) { +TAsyncStatus TConfigClient::BootstrapCluster(const std::string& selfAssemblyUUID, const TBootstrapClusterSettings& settings) { return Impl_->BootstrapCluster(selfAssemblyUUID, settings); } From f6ff75050038c5324e1d7c28d76004e59eb6e8ac Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 16 Feb 2025 13:27:13 +0000 Subject: [PATCH 20/50] WIP --- .../lib/ydb_cli/commands/ydb_cluster.cpp | 2 +- .../ydb_cli/commands/ydb_dynamic_config.cpp | 4 ++-- .../ydb_cli/commands/ydb_storage_config.cpp | 8 +++---- .../ydb-cpp-sdk/client/config/config.h | 22 +++++++++---------- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp b/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp index e0e6a8c48399..1ab0cec51741 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp @@ -38,7 +38,7 @@ void TCommandClusterBootstrap::Parse(TConfig& config) { int TCommandClusterBootstrap::Run(TConfig& config) { auto driver = std::make_unique(CreateDriver(config)); - NYdb::NStorageConfig::TStorageConfigClient client(*driver); + NYdb::NConfig::TConfigClient client(*driver); auto result = client.BootstrapCluster(SelfAssemblyUUID).GetValueSync(); NStatusHelpers::ThrowOnErrorOrPrintIssues(result); return EXIT_SUCCESS; diff --git a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp index 1c113913a424..387ba381d221 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp @@ -181,9 +181,9 @@ void TCommandConfigReplace::Parse(TConfig& config) { int TCommandConfigReplace::Run(TConfig& config) { std::unique_ptr driver = std::make_unique(CreateDriver(config)); - auto client = NYdb::NStorageConfig::TStorageConfigClient(*driver); + auto client = NYdb::NConfig::TConfigClient(*driver); - NYdb::NStorageConfig::TReplaceConfigSettings settings; + NYdb::NConfig::TReplaceConfigSettings settings; if (Force) { settings.AllowIncorrectVersion(); diff --git a/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp b/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp index 166bb3c2341c..62c4bf9dba5e 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp @@ -62,11 +62,11 @@ void TCommandStorageConfigFetch::Parse(TConfig& config) { int TCommandStorageConfigFetch::Run(TConfig& config) { auto driver = std::make_unique(CreateDriver(config)); - auto client = NYdb::NStorageConfig::TStorageConfigClient(*driver); + auto client = NYdb::NConfig::TConfigClient(*driver); auto result = client.FetchConfig(DedicatedStorageSection, DedicatedClusterSection).GetValueSync(); NStatusHelpers::ThrowOnError(result); - const auto& clusterConfig = result.GetConfig(); + const auto& clusterConfig = result.GetMainConfig(); const auto& storageConfig = result.GetStorageConfig(); if (!clusterConfig.empty()) { @@ -141,8 +141,8 @@ void TCommandStorageConfigReplace::Parse(TConfig& config) { int TCommandStorageConfigReplace::Run(TConfig& config) { std::unique_ptr driver = std::make_unique(CreateDriver(config)); - auto client = NYdb::NStorageConfig::TStorageConfigClient(*driver); - NYdb::NStorageConfig::TReplaceConfigSettings settings; + auto client = NYdb::NConfig::TConfigClient(*driver); + NYdb::NConfig::TReplaceConfigSettings settings; settings .SwitchDedicatedStorageSection(SwitchDedicatedStorageSection) .DedicatedConfigMode(DedicatedConfigMode); diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/config/config.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/config/config.h index a2f06d37e7e3..f829af09c5a6 100644 --- a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/config/config.h +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/config/config.h @@ -14,24 +14,24 @@ namespace NYdb::inline V3::NConfig { struct TFetchConfigResult : public TStatus { TFetchConfigResult( TStatus&& status, - std::string&& config, + std::string&& main_config, std::string&& storage_config) : TStatus(std::move(status)) - , Config_(std::move(config)) - , Config_(std::move(storage_config)) + , MainConfig_(std::move(main_config)) + , StorageConfig_(std::move(storage_config)) {} - const std::string& GetConfig() const { - return Config_; + const std::string& GetMainConfig() const { + return MainConfig_; } - const std::string& GetConfig() const { - return Config_; + const std::string& GetStorageConfig() const { + return StorageConfig_; } private: - std::string Config_; - std::string Config_; + std::string MainConfig_; + std::string StorageConfig_; }; using TAsyncFetchConfigResult = NThreading::TFuture; @@ -58,8 +58,8 @@ class TConfigClient { // Replace config TAsyncStatus ReplaceConfig( - const std::optional& yaml_config, - const std::optional& storage_yaml_config, + const std::optional& main_config, + const std::optional& storage_config, const TReplaceConfigSettings& settings = {}); // Fetch current cluster storage config From c5d3a962252fecb9f6fa4880c6f6bb763b11fd62 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 16 Feb 2025 14:01:01 +0000 Subject: [PATCH 21/50] WIP --- ydb/public/sdk/cpp/src/client/ya.make | 28 +++++++++++++-------------- ydb/services/config/ut/ya.make | 6 +++--- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/ydb/public/sdk/cpp/src/client/ya.make b/ydb/public/sdk/cpp/src/client/ya.make index 65e4578e55fa..89be03b388dc 100644 --- a/ydb/public/sdk/cpp/src/client/ya.make +++ b/ydb/public/sdk/cpp/src/client/ya.make @@ -1,7 +1,19 @@ RECURSE( - bsconfig + cms + common_client + common_client/impl + config + coordination + datastreams + debug + discovery draft + driver + export + extension_common extensions + federated_topic + federated_topic/impl helpers impl/ydb_endpoints impl/ydb_internal/common @@ -13,19 +25,6 @@ RECURSE( impl/ydb_internal/thread_pool impl/ydb_internal/value_helpers impl/ydb_stats - resources - cms - common_client - common_client/impl - coordination - datastreams - debug - discovery - driver - export - extension_common - federated_topic - federated_topic/impl import operation params @@ -35,6 +34,7 @@ RECURSE( query query/impl rate_limiter + resources result scheme table diff --git a/ydb/services/config/ut/ya.make b/ydb/services/config/ut/ya.make index 94e955e17300..6b07c71b29a0 100644 --- a/ydb/services/config/ut/ya.make +++ b/ydb/services/config/ut/ya.make @@ -1,16 +1,16 @@ -UNITTEST_FOR(ydb/services/bsconfig) +UNITTEST_FOR(ydb/services/config) SIZE(MEDIUM) SRCS( - bsconfig_ut.cpp + config_ut.cpp ) PEERDIR( library/cpp/logger ydb/core/protos ydb/core/testlib/default - ydb/services/bsconfig + ydb/services/config ) YQL_LAST_ABI_VERSION() From 98d2a0d9cab814ca524978783db38b17f0f994cf Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 16 Feb 2025 14:17:52 +0000 Subject: [PATCH 22/50] WIP --- ydb/core/grpc_services/rpc_config.cpp | 4 ++-- .../{service_bsconfig.h => service_config.h} | 4 ++-- ydb/core/jaeger_tracing/request_discriminator.cpp | 6 +++--- ydb/core/jaeger_tracing/request_discriminator.h | 6 +++--- ydb/services/config/grpc_service.cpp | 10 +++++----- 5 files changed, 15 insertions(+), 15 deletions(-) rename ydb/core/grpc_services/{service_bsconfig.h => service_config.h} (63%) diff --git a/ydb/core/grpc_services/rpc_config.cpp b/ydb/core/grpc_services/rpc_config.cpp index ff2c81b36a46..703da58eeedf 100644 --- a/ydb/core/grpc_services/rpc_config.cpp +++ b/ydb/core/grpc_services/rpc_config.cpp @@ -296,11 +296,11 @@ class TFetchStorageConfigRequest : public TBSConfigRequestGrpc p, const IFacilityProvider&) { +void DoReplaceConfig(std::unique_ptr p, const IFacilityProvider&) { TActivationContext::AsActorContext().Register(new TReplaceStorageConfigRequest(p.release())); } -void DoFetchBSConfig(std::unique_ptr p, const IFacilityProvider&) { +void DoFetchConfig(std::unique_ptr p, const IFacilityProvider&) { TActivationContext::AsActorContext().Register(new TFetchStorageConfigRequest(p.release())); } diff --git a/ydb/core/grpc_services/service_bsconfig.h b/ydb/core/grpc_services/service_config.h similarity index 63% rename from ydb/core/grpc_services/service_bsconfig.h rename to ydb/core/grpc_services/service_config.h index c09d5b40db4a..5b665ed30b35 100644 --- a/ydb/core/grpc_services/service_bsconfig.h +++ b/ydb/core/grpc_services/service_config.h @@ -7,9 +7,9 @@ namespace NKikimr::NGRpcService { class IRequestOpCtx; class IFacilityProvider; - void DoReplaceBSConfig(std::unique_ptr p, const IFacilityProvider&); + void DoReplaceConfig(std::unique_ptr p, const IFacilityProvider&); - void DoFetchBSConfig(std::unique_ptr p, const IFacilityProvider&); + void DoFetchConfig(std::unique_ptr p, const IFacilityProvider&); void DoBootstrapCluster(std::unique_ptr p, const IFacilityProvider&); diff --git a/ydb/core/jaeger_tracing/request_discriminator.cpp b/ydb/core/jaeger_tracing/request_discriminator.cpp index 73f46bec1321..51e35f035f02 100644 --- a/ydb/core/jaeger_tracing/request_discriminator.cpp +++ b/ydb/core/jaeger_tracing/request_discriminator.cpp @@ -66,9 +66,9 @@ extern const THashMap NameToRequestType = { {"RateLimiter.DescribeResource", ERequestType::RATELIMITER_DESCRIBE_RESOURCE}, {"RateLimiter.AcquireResource", ERequestType::RATELIMITER_ACQUIRE_RESOURCE}, - {"BSConfig.ReplaceStorageConfig", ERequestType::BSCONFIG_REPLACESTORAGECONFIG}, - {"BSConfig.FetchStorageConfig", ERequestType::BSCONFIG_FETCHSTORAGECONFIG}, - {"BSConfig.Bootstrap", ERequestType::BSCONFIG_BOOTSTRAP}, + {"Config.ReplaceConfig", ERequestType::CONFIG_REPLACECONFIG}, + {"Config.FetchConfig", ERequestType::CONFIG_FETCHCONFIG}, + {"Config.Bootstrap", ERequestType::CONFIG_BOOTSTRAP}, {"Topic.StreamWrite", ERequestType::TOPIC_STREAMWRITE}, {"Topic.StreamWrite.Init", ERequestType::TOPIC_STREAMWRITE_INIT}, diff --git a/ydb/core/jaeger_tracing/request_discriminator.h b/ydb/core/jaeger_tracing/request_discriminator.h index c02f08281cc4..42f46a0110cd 100644 --- a/ydb/core/jaeger_tracing/request_discriminator.h +++ b/ydb/core/jaeger_tracing/request_discriminator.h @@ -71,9 +71,9 @@ enum class ERequestType: size_t { RATELIMITER_DESCRIBE_RESOURCE, RATELIMITER_ACQUIRE_RESOURCE, - BSCONFIG_REPLACESTORAGECONFIG, - BSCONFIG_FETCHSTORAGECONFIG, - BSCONFIG_BOOTSTRAP, + CONFIG_REPLACECONFIG, + CONFIG_FETCHCONFIG, + CONFIG_BOOTSTRAP, PING_GRPC, PING_PROXY, diff --git a/ydb/services/config/grpc_service.cpp b/ydb/services/config/grpc_service.cpp index 4e7e257c8169..5198fe9215cd 100644 --- a/ydb/services/config/grpc_service.cpp +++ b/ydb/services/config/grpc_service.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include "ydb/library/grpc/server/grpc_method_setup.h" @@ -26,11 +26,11 @@ void TConfigGRpcService::SetupIncomingRequests(NYdbGrpc::TLoggerPtr logger) { auto getCounterBlock = NGRpcService::CreateCounterCb(Counters, ActorSystem); #define SETUP_BS_METHOD(methodName, method, rlMode, requestType) \ - SETUP_METHOD(methodName, method, rlMode, requestType, Config, bsconfig) + SETUP_METHOD(methodName, method, rlMode, requestType, Config, config) - SETUP_BS_METHOD(ReplaceConfig, DoReplaceBSConfig, Rps, BSCONFIG_REPLACESTORAGECONFIG); - SETUP_BS_METHOD(FetchConfig, DoFetchBSConfig, Rps, BSCONFIG_FETCHSTORAGECONFIG); - SETUP_BS_METHOD(BootstrapCluster, DoBootstrapCluster, Rps, BSCONFIG_BOOTSTRAP); + SETUP_BS_METHOD(ReplaceConfig, DoReplaceConfig, Rps, CONFIG_REPLACECONFIG); + SETUP_BS_METHOD(FetchConfig, DoFetchConfig, Rps, CONFIG_FETCHCONFIG); + SETUP_BS_METHOD(BootstrapCluster, DoBootstrapCluster, Rps, CONFIG_BOOTSTRAP); #undef SETUP_BS_METHOD } From 9fc81bc3c03d9571596352457f169f1630d8e7ee Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 16 Feb 2025 14:32:45 +0000 Subject: [PATCH 23/50] WIP --- ydb/services/config/ut/ya.make | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/services/config/ut/ya.make b/ydb/services/config/ut/ya.make index 6b07c71b29a0..1004716648e1 100644 --- a/ydb/services/config/ut/ya.make +++ b/ydb/services/config/ut/ya.make @@ -3,7 +3,7 @@ UNITTEST_FOR(ydb/services/config) SIZE(MEDIUM) SRCS( - config_ut.cpp + bsconfig_ut.cpp ) PEERDIR( From c8b67a9a468745f4772456f6022d378d71437691 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 16 Feb 2025 15:20:44 +0000 Subject: [PATCH 24/50] WIP --- .../sdk/cpp/src/client/config/config.cpp | 58 ++++++++++--------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/ydb/public/sdk/cpp/src/client/config/config.cpp b/ydb/public/sdk/cpp/src/client/config/config.cpp index cc960bff550b..a012612e0133 100644 --- a/ydb/public/sdk/cpp/src/client/config/config.cpp +++ b/ydb/public/sdk/cpp/src/client/config/config.cpp @@ -15,30 +15,32 @@ class TConfigClient::TImpl : public TClientImplCommon { } TAsyncStatus ReplaceConfig( - const std::optional& yaml_config, - const std::optional& storage_yaml_config, + const std::optional& main_config, + const std::optional& storage_config, const TReplaceConfigSettings& settings) { auto request = MakeRequest(); - if (yaml_config) { - request.set_main_config(*yaml_config); - } + Y_UNUSED(main_config, storage_config, settings); // FIXME - if (storage_yaml_config) { - request.set_storage_config(*storage_yaml_config); - } + // if (yaml_config) { + // request.set_main_config(*yaml_config); + // } - if (settings.SwitchDedicatedStorageSection_) { - request.set_switch_dedicated_storage_section(*settings.SwitchDedicatedStorageSection_); - } + // if (storage_yaml_config) { + // request.set_storage_config(*storage_yaml_config); + // } - request.set_dedicated_config_mode(settings.DedicatedConfigMode_); - request.set_dry_run(settings.DryRun_); - request.set_allow_unknown_fields(settings.AllowUnknownFields_); - request.set_allow_absent_database(settings.AllowAbsentDatabase_); - request.set_allow_incorrect_version(settings.AllowIncorrectVersion_); - request.set_allow_incorrect_cluster(settings.AllowIncorrectCluster_); + // if (settings.SwitchDedicatedStorageSection_) { + // request.set_switch_dedicated_storage_section(*settings.SwitchDedicatedStorageSection_); + // } + + // request.set_dedicated_config_mode(settings.DedicatedConfigMode_); + // request.set_dry_run(settings.DryRun_); + // request.set_allow_unknown_fields(settings.AllowUnknownFields_); + // request.set_allow_absent_database(settings.AllowAbsentDatabase_); + // request.set_allow_incorrect_version(settings.AllowIncorrectVersion_); + // request.set_allow_incorrect_cluster(settings.AllowIncorrectCluster_); return RunSimple( std::move(request), @@ -50,21 +52,23 @@ class TConfigClient::TImpl : public TClientImplCommon { const TFetchConfigSettings& settings) { auto request = MakeOperationRequest(settings); - if (dedicated_storage_section) { - request.set_dedicated_storage_section(true); - } - if (dedicated_cluster_section) { - request.set_dedicated_cluster_section(true); - } + Y_UNUSED(dedicated_storage_section, dedicated_cluster_section, settings); // FIXME + // if (dedicated_storage_section) { + // request.set_dedicated_storage_section(true); + // } + // if (dedicated_cluster_section) { + // request.set_dedicated_cluster_section(true); + // } auto promise = NThreading::NewPromise(); auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { NYdb::TStringType config; NYdb::TStringType storage_config; - if (Ydb::Config::FetchConfigResult result; any && any->UnpackTo(&result)) { - config = result.main_config(); - storage_config = result.storage_config(); - } + Y_UNUSED(any); + // if (Ydb::Config::FetchConfigResult result; any && any->UnpackTo(&result)) { + // config = result.main_config(); + // storage_config = result.storage_config(); + // } TFetchConfigResult val(TStatus(std::move(status)), std::string{std::move(config)}, std::string{std::move(storage_config)}); From a6b181c4ecf68aa0f84a7625d02fe243b5e75e6e Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 16 Feb 2025 16:20:17 +0000 Subject: [PATCH 25/50] WIP --- ydb/services/config/bsconfig_ut.cpp | 108 +++++++++++++++------------- 1 file changed, 57 insertions(+), 51 deletions(-) diff --git a/ydb/services/config/bsconfig_ut.cpp b/ydb/services/config/bsconfig_ut.cpp index eff2f5a763bb..9ed6265ea534 100644 --- a/ydb/services/config/bsconfig_ut.cpp +++ b/ydb/services/config/bsconfig_ut.cpp @@ -73,7 +73,7 @@ class TBasicKikimrWithGrpcAndRootSchema { } ServerSettings->Formats = new NKikimr::TFormatFactory; ServerSettings->FeatureFlags = appConfig.GetFeatureFlags(); - ServerSettings->RegisterGrpcService("bsconfig"); + ServerSettings->RegisterGrpcService("bsconfig"); Server_.Reset(new NKikimr::Tests::TServer(*ServerSettings)); Tenants_.Reset(new NKikimr::Tests::TTenants(Server_)); @@ -158,73 +158,79 @@ TString NormalizeYaml(const TString& yaml) { return normalized.Str(); } -Y_UNIT_TEST_SUITE(BSConfigGRPCService) { +Y_UNIT_TEST_SUITE(ConfigGRPCService) { template void AdjustCtxForDB(TCtx &ctx) { ctx.AddMetadata(NYdb::YDB_AUTH_TICKET_HEADER, "root@builtin"); } - void ReplaceStorageConfig(auto &channel, std::optional yamlConfig, std::optional storageYamlConfig, + void ReplaceConfig(auto &channel, std::optional yamlConfig, std::optional storageYamlConfig, std::optional switchDedicatedStorageSection, bool dedicatedConfigMode) { - std::unique_ptr stub; - stub = Ydb::Config::V1::BSConfigService::NewStub(channel); - - Ydb::Config::ReplaceStorageConfigRequest request; - if (yamlConfig) { - request.set_yaml_config(*yamlConfig); - } - if (storageYamlConfig) { - request.set_storage_yaml_config(*storageYamlConfig); - } - if (switchDedicatedStorageSection) { - request.set_switch_dedicated_storage_section(*switchDedicatedStorageSection); - } - request.set_dedicated_config_mode(dedicatedConfigMode); - - Ydb::Config::ReplaceStorageConfigResponse response; - Ydb::Config::ReplaceStorageConfigResult result; - - grpc::ClientContext replaceStorageConfigCtx; - AdjustCtxForDB(replaceStorageConfigCtx); - stub->ReplaceStorageConfig(&replaceStorageConfigCtx, request, &response); + std::unique_ptr stub; + stub = Ydb::Config::V1::ConfigService::NewStub(channel); + + Ydb::Config::ReplaceConfigRequest request; + Y_UNUSED(yamlConfig, storageYamlConfig, switchDedicatedStorageSection, dedicatedConfigMode); + // FIXME + // if (yamlConfig) { + // request.set_main_config(*yamlConfig); + // } + // if (storageYamlConfig) { + // request.set_storage_config(*storageYamlConfig); + // } + // if (switchDedicatedStorageSection) { + // request.set_switch_dedicated_storage_section(*switchDedicatedStorageSection); + // } + // request.set_dedicated_config_mode(dedicatedConfigMode); + + Ydb::Config::ReplaceConfigResponse response; + Ydb::Config::ReplaceConfigResult result; + + grpc::ClientContext replaceConfigCtx; + AdjustCtxForDB(replaceConfigCtx); + stub->ReplaceConfig(&replaceConfigCtx, request, &response); UNIT_ASSERT_CHECK_STATUS(response.operation(), Ydb::StatusIds::SUCCESS); Cerr << "response: " << response.operation().result().DebugString() << Endl; response.operation().result().UnpackTo(&result); } - void FetchStorageConfig(auto& channel, bool dedicatedStorageSection, bool dedicatedClusterSection, + void FetchConfig(auto& channel, bool dedicatedStorageSection, bool dedicatedClusterSection, std::optional& yamlConfig, std::optional& storageYamlConfig) { - std::unique_ptr stub; - stub = Ydb::Config::V1::BSConfigService::NewStub(channel); + std::unique_ptr stub; + stub = Ydb::Config::V1::ConfigService::NewStub(channel); - Ydb::Config::FetchStorageConfigRequest request; - request.set_dedicated_storage_section(dedicatedStorageSection); - request.set_dedicated_cluster_section(dedicatedClusterSection); + Ydb::Config::FetchConfigRequest request; + Y_UNUSED(dedicatedStorageSection, dedicatedClusterSection); + // FIXME + // request.set_dedicated_storage_section(dedicatedStorageSection); + // request.set_dedicated_cluster_section(dedicatedClusterSection); - Ydb::Config::FetchStorageConfigResponse response; - Ydb::Config::FetchStorageConfigResult result; + Ydb::Config::FetchConfigResponse response; + Ydb::Config::FetchConfigResult result; - grpc::ClientContext fetchStorageConfigCtx; - AdjustCtxForDB(fetchStorageConfigCtx); - stub->FetchStorageConfig(&fetchStorageConfigCtx, request, &response); + grpc::ClientContext fetchConfigCtx; + AdjustCtxForDB(fetchConfigCtx); + stub->FetchConfig(&fetchConfigCtx, request, &response); UNIT_ASSERT_CHECK_STATUS(response.operation(), Ydb::StatusIds::SUCCESS); response.operation().result().UnpackTo(&result); - if (result.has_yaml_config()) { - yamlConfig.emplace(result.yaml_config()); - } else { - yamlConfig.reset(); - } - - if (result.has_storage_yaml_config()) { - storageYamlConfig.emplace(result.storage_yaml_config()); - } else { - storageYamlConfig.reset(); - } + Y_UNUSED(yamlConfig, storageYamlConfig); + // FIXME + // if (result.has_main_config()) { + // yamlConfig.emplace(result.main_config()); + // } else { + // yamlConfig.reset(); + // } + + // if (result.has_storage_config()) { + // storageYamlConfig.emplace(result.storage_config()); + // } else { + // storageYamlConfig.reset(); + // } } - Y_UNIT_TEST(ReplaceStorageConfig) { + Y_UNIT_TEST(ReplaceConfig) { TKikimrWithGrpcAndRootSchema server; TString yamlConfig = R"( metadata: @@ -263,18 +269,18 @@ selector_config: [] host_config_id: 2 )"; TString yamlConfigExpected = SubstGlobalCopy(yamlConfig, "version: 0", "version: 1"); - ReplaceStorageConfig(server.GetChannel(), yamlConfig, std::nullopt, std::nullopt, false); + ReplaceConfig(server.GetChannel(), yamlConfig, std::nullopt, std::nullopt, false); std::optional yamlConfigFetched, storageYamlConfigFetched; - FetchStorageConfig(server.GetChannel(), false, false, yamlConfigFetched, storageYamlConfigFetched); + FetchConfig(server.GetChannel(), false, false, yamlConfigFetched, storageYamlConfigFetched); UNIT_ASSERT(yamlConfigFetched); UNIT_ASSERT(!storageYamlConfigFetched); UNIT_ASSERT_VALUES_EQUAL(yamlConfigExpected, *yamlConfigFetched); } - Y_UNIT_TEST(FetchStorageConfig) { + Y_UNIT_TEST(FetchConfig) { TKikimrWithGrpcAndRootSchema server; std::optional yamlConfigFetched, storageYamlConfigFetched; - FetchStorageConfig(server.GetChannel(), false, false, yamlConfigFetched, storageYamlConfigFetched); + FetchConfig(server.GetChannel(), false, false, yamlConfigFetched, storageYamlConfigFetched); UNIT_ASSERT(!yamlConfigFetched); UNIT_ASSERT(!storageYamlConfigFetched); } From 6ff72aed0f562d6398bbd71cf2c117d89d757f26 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 16 Feb 2025 18:31:33 +0000 Subject: [PATCH 26/50] WIP --- .../library/clients/kikimr_config_client.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/ydb/tests/library/clients/kikimr_config_client.py b/ydb/tests/library/clients/kikimr_config_client.py index 6dbde343ca0a..30af9bf7f1bf 100644 --- a/ydb/tests/library/clients/kikimr_config_client.py +++ b/ydb/tests/library/clients/kikimr_config_client.py @@ -7,14 +7,14 @@ import grpc import logging -from ydb.public.api.grpc import ydb_bsconfig_v1_pb2_grpc as grpc_server -from ydb.public.api.protos import ydb_bsconfig_pb2 as bsconfig_api +from ydb.public.api.grpc import ydb_config_v1_pb2_grpc as grpc_server +from ydb.public.api.protos import ydb_config_pb2 as config_api logger = logging.getLogger() -def bsconfig_client_factory(server, port, cluster=None, retry_count=1): - return BSConfigClient( +def config_client_factory(server, port, cluster=None, retry_count=1): + return ConfigClient( server, port, cluster=cluster, retry_count=retry_count ) @@ -24,7 +24,7 @@ def channels_list(): return os.getenv('CHANNELS_LIST', '') -class BSConfigClient(object): +class ConfigClient(object): def __init__(self, server, port, cluster=None, retry_count=1): self.server = server self.port = port @@ -37,7 +37,7 @@ def __init__(self, server, port, cluster=None, retry_count=1): ('grpc.max_send_message_length', 64 * 10 ** 6) ] self._channel = grpc.insecure_channel("%s:%s" % (self.server, self.port), options=self._options) - self._stub = grpc_server.BSConfigServiceStub(self._channel) + self._stub = grpc_server.ConfigServiceStub(self._channel) def _get_invoke_callee(self, method): return getattr(self._stub, method) @@ -57,20 +57,20 @@ def invoke(self, request, method): time.sleep(self.__retry_sleep_seconds) def replace_storage_config(self, yaml_config, storage_yaml_config=None): - request = bsconfig_api.ReplaceStorageConfigRequest() + request = config_api.ReplaceConfigRequest() request.yaml_config = yaml_config if storage_yaml_config is not None: request.storage_yaml_config = storage_yaml_config - return self.invoke(request, 'ReplaceStorageConfig') + return self.invoke(request, 'ReplaceConfig') def fetch_storage_config(self, dedicated_storage_section=False, dedicated_cluster_section=False): - request = bsconfig_api.FetchStorageConfigRequest() + request = config_api.FetchConfigRequest() request.dedicated_storage_section = dedicated_storage_section request.dedicated_cluster_section = dedicated_cluster_section - return self.invoke(request, 'FetchStorageConfig') + return self.invoke(request, 'FetchConfig') def bootstrap_cluster(self, self_assembly_uuid): - request = bsconfig_api.BootstrapClusterRequest() + request = config_api.BootstrapClusterRequest() request.self_assembly_uuid = self_assembly_uuid return self.invoke(request, 'BootstrapCluster') From dbbc677f60102e9d7768ae2f9eaf970594ef4b9c Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 16 Feb 2025 20:36:06 +0000 Subject: [PATCH 27/50] WIP --- .../yaml_config/yaml_config_parser.cpp | 3 +- ydb/services/config/bsconfig_ut.cpp | 95 ++++++++++++------- 2 files changed, 63 insertions(+), 35 deletions(-) diff --git a/ydb/library/yaml_config/yaml_config_parser.cpp b/ydb/library/yaml_config/yaml_config_parser.cpp index a735ecb8efd4..17611786046b 100644 --- a/ydb/library/yaml_config/yaml_config_parser.cpp +++ b/ydb/library/yaml_config/yaml_config_parser.cpp @@ -1421,8 +1421,7 @@ namespace NKikimr::NYaml { void TransformProtoConfig(TTransformContext& ctx, NKikimrConfig::TAppConfig& config, NKikimrConfig::TEphemeralInputFields& ephemeralConfig, bool relaxed) { PrepareHosts(ephemeralConfig); MoveFields(ctx, config, ephemeralConfig); - // FIXME - //ApplyDefaultConfigs(ctx, config, ephemeralConfig); + ApplyDefaultConfigs(ctx, config, ephemeralConfig); PrepareNameserviceConfig(config, ephemeralConfig); PrepareStaticGroup(ctx, config, ephemeralConfig); PrepareBlobStorageConfig(config, ephemeralConfig); diff --git a/ydb/services/config/bsconfig_ut.cpp b/ydb/services/config/bsconfig_ut.cpp index 9ed6265ea534..10f42e4d390b 100644 --- a/ydb/services/config/bsconfig_ut.cpp +++ b/ydb/services/config/bsconfig_ut.cpp @@ -165,24 +165,45 @@ Y_UNIT_TEST_SUITE(ConfigGRPCService) { ctx.AddMetadata(NYdb::YDB_AUTH_TICKET_HEADER, "root@builtin"); } - void ReplaceConfig(auto &channel, std::optional yamlConfig, std::optional storageYamlConfig, - std::optional switchDedicatedStorageSection, bool dedicatedConfigMode) { + void ReplaceConfig( + auto &channel, + std::optional mainConfig, + std::optional storageConfig, + std::optional switchDedicatedStorageSection, + bool dedicatedConfigMode) { + std::unique_ptr stub; stub = Ydb::Config::V1::ConfigService::NewStub(channel); Ydb::Config::ReplaceConfigRequest request; - Y_UNUSED(yamlConfig, storageYamlConfig, switchDedicatedStorageSection, dedicatedConfigMode); - // FIXME - // if (yamlConfig) { - // request.set_main_config(*yamlConfig); - // } - // if (storageYamlConfig) { - // request.set_storage_config(*storageYamlConfig); - // } - // if (switchDedicatedStorageSection) { - // request.set_switch_dedicated_storage_section(*switchDedicatedStorageSection); - // } - // request.set_dedicated_config_mode(dedicatedConfigMode); + + if (!dedicatedConfigMode && !switchDedicatedStorageSection) { + if (mainConfig) { + request.set_replace(*mainConfig); + } + } else if (dedicatedConfigMode && !switchDedicatedStorageSection) { + auto& replace = *request.mutable_replace_with_dedicated_storage_section(); + if (mainConfig) { + replace.set_main_config(*mainConfig); + } + if (storageConfig) { + replace.set_storage_config(*storageConfig); + } + } else if (switchDedicatedStorageSection && *switchDedicatedStorageSection) { + auto& replace = *request.mutable_replace_enable_dedicated_storage_section(); + if (mainConfig) { + replace.set_main_config(*mainConfig); + } + if (storageConfig) { + replace.set_storage_config(*storageConfig); + } + } else if (switchDedicatedStorageSection && !*switchDedicatedStorageSection) { + if (mainConfig) { + request.set_replace_disable_dedicated_storage_section(*mainConfig); + } + } else { + Y_ABORT("invariant violation"); + } Ydb::Config::ReplaceConfigResponse response; Ydb::Config::ReplaceConfigResult result; @@ -195,16 +216,22 @@ Y_UNIT_TEST_SUITE(ConfigGRPCService) { response.operation().result().UnpackTo(&result); } - void FetchConfig(auto& channel, bool dedicatedStorageSection, bool dedicatedClusterSection, - std::optional& yamlConfig, std::optional& storageYamlConfig) { + void FetchConfig( + auto& channel, + bool dedicatedStorageSection, + bool dedicatedClusterSection, + std::optional& mainConfig, + std::optional& storageConfig) { std::unique_ptr stub; stub = Ydb::Config::V1::ConfigService::NewStub(channel); Ydb::Config::FetchConfigRequest request; - Y_UNUSED(dedicatedStorageSection, dedicatedClusterSection); - // FIXME - // request.set_dedicated_storage_section(dedicatedStorageSection); - // request.set_dedicated_cluster_section(dedicatedClusterSection); + + auto& all = *request.mutable_all(); + + if (dedicatedStorageSection || dedicatedClusterSection) { + all.mutable_detach_storage_config_section(); + } Ydb::Config::FetchConfigResponse response; Ydb::Config::FetchConfigResult result; @@ -215,19 +242,21 @@ Y_UNIT_TEST_SUITE(ConfigGRPCService) { UNIT_ASSERT_CHECK_STATUS(response.operation(), Ydb::StatusIds::SUCCESS); response.operation().result().UnpackTo(&result); - Y_UNUSED(yamlConfig, storageYamlConfig); - // FIXME - // if (result.has_main_config()) { - // yamlConfig.emplace(result.main_config()); - // } else { - // yamlConfig.reset(); - // } - - // if (result.has_storage_config()) { - // storageYamlConfig.emplace(result.storage_config()); - // } else { - // storageYamlConfig.reset(); - // } + std::optional rcvMainConfig; + std::optional rcvStorageConfig; + + for (auto& entry : result.config()) { + if (entry.identity().type_case() == Ydb::Config::ConfigIdentity::TypeCase::kMain) { + rcvMainConfig = entry.config(); + } + + if (entry.identity().type_case() == Ydb::Config::ConfigIdentity::TypeCase::kStorage) { + rcvStorageConfig = entry.config(); + } + } + + mainConfig = rcvMainConfig; + storageConfig = rcvStorageConfig; } Y_UNIT_TEST(ReplaceConfig) { From dbb88dc440aa8204abd5b94d2dc4380c415143c2 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Mon, 17 Feb 2025 07:20:02 +0000 Subject: [PATCH 28/50] WIP --- .../config/test_config_with_metadata.py | 26 ++++++++++--------- .../library/clients/kikimr_config_client.py | 11 ++++---- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/ydb/tests/functional/config/test_config_with_metadata.py b/ydb/tests/functional/config/test_config_with_metadata.py index f4772b197510..2ccb47ddf532 100644 --- a/ydb/tests/functional/config/test_config_with_metadata.py +++ b/ydb/tests/functional/config/test_config_with_metadata.py @@ -9,12 +9,12 @@ from ydb.tests.library.harness.util import LogLevels from ydb.tests.library.clients.kikimr_http_client import SwaggerClient from ydb.tests.library.harness.kikimr_runner import KiKiMR -from ydb.tests.library.clients.kikimr_config_client import BSConfigClient +from ydb.tests.library.clients.kikimr_config_client import ConfigClient from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator from ydb.tests.library.kv.helpers import create_kv_tablets_and_wait_for_start from ydb.public.api.protos.ydb_status_codes_pb2 import StatusIds -import ydb.public.api.protos.ydb_bsconfig_pb2 as bsconfig +import ydb.public.api.protos.ydb_config_pb2 as config logger = logging.getLogger(__name__) @@ -50,7 +50,7 @@ def setup_class(cls): host = cls.cluster.nodes[1].host grpc_port = cls.cluster.nodes[1].port cls.swagger_client = SwaggerClient(host, cls.cluster.nodes[1].mon_port) - cls.bsconfig_client = BSConfigClient(host, grpc_port) + cls.config_client = ConfigClient(host, grpc_port) @classmethod def teardown_class(cls): @@ -141,7 +141,7 @@ def setup_class(cls): use_in_memory_pdisks=False, use_config_store=True, separate_node_configs=True, - extra_grpc_services=['bsconfig'], + extra_grpc_services=['config'], metadata_section=cls.metadata_section, additional_log_configs={'BS_NODE': LogLevels.DEBUG}, ) @@ -151,7 +151,7 @@ def setup_class(cls): host = cls.cluster.nodes[1].host grpc_port = cls.cluster.nodes[1].port cls.swagger_client = SwaggerClient(host, cls.cluster.nodes[1].mon_port) - cls.bsconfig_client = BSConfigClient(host, grpc_port) + cls.config_client = ConfigClient(host, grpc_port) def test_cluster_works_with_auto_conf_dir(self): table_path = '/Root/mydb/mytable_auto_conf' @@ -174,16 +174,18 @@ def test_config_stored_in_config_store(self): initial_config['metadata']['version'] = initial_version config_yaml = yaml.dump(initial_config) - replace_storage_config_response = self.bsconfig_client.replace_storage_config(config_yaml) - assert_that(replace_storage_config_response.operation.status == StatusIds.SUCCESS) + replace_config_response = self.config_client.replace_config(config_yaml) + assert_that(replace_config_response.operation.status == StatusIds.SUCCESS) - fetch_storage_config_response = self.bsconfig_client.fetch_storage_config() - assert_that(fetch_storage_config_response.operation.status == StatusIds.SUCCESS) + fetch_config_response = self.config_client.fetch_config() + assert_that(fetch_config_response.operation.status == StatusIds.SUCCESS) - result = bsconfig.FetchStorageConfigResult() - fetch_storage_config_response.operation.result.Unpack(result) + result = config.FetchConfigResult() + fetch_config_response.operation.result.Unpack(result) - fetched_config = result.yaml_config + assert_that(result.config is not None) + assert_that(len(result.config) == 1) + fetched_config = result.config[0].config parsed_fetched_config = yaml.safe_load(fetched_config) assert_that(parsed_fetched_config is not None) assert_that(parsed_fetched_config.get('metadata') is not None) diff --git a/ydb/tests/library/clients/kikimr_config_client.py b/ydb/tests/library/clients/kikimr_config_client.py index 30af9bf7f1bf..f1b0af7c2984 100644 --- a/ydb/tests/library/clients/kikimr_config_client.py +++ b/ydb/tests/library/clients/kikimr_config_client.py @@ -56,17 +56,18 @@ def invoke(self, request, method): time.sleep(self.__retry_sleep_seconds) - def replace_storage_config(self, yaml_config, storage_yaml_config=None): + def replace_config(self, yaml_config, storage_yaml_config=None): request = config_api.ReplaceConfigRequest() - request.yaml_config = yaml_config + request.replace = yaml_config if storage_yaml_config is not None: request.storage_yaml_config = storage_yaml_config return self.invoke(request, 'ReplaceConfig') - def fetch_storage_config(self, dedicated_storage_section=False, dedicated_cluster_section=False): + def fetch_config(self, dedicated_storage_section=False, dedicated_cluster_section=False): request = config_api.FetchConfigRequest() - request.dedicated_storage_section = dedicated_storage_section - request.dedicated_cluster_section = dedicated_cluster_section + # FIXME + # request.dedicated_storage_section = dedicated_storage_section + # request.dedicated_cluster_section = dedicated_cluster_section return self.invoke(request, 'FetchConfig') def bootstrap_cluster(self, self_assembly_uuid): From 0302eada89b427c027ce1c1868d05779f72a89d7 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Mon, 17 Feb 2025 13:21:34 +0000 Subject: [PATCH 29/50] WIP --- .../config/test_config_with_metadata.py | 2 +- .../library/clients/kikimr_config_client.py | 26 +++++++++++++------ 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/ydb/tests/functional/config/test_config_with_metadata.py b/ydb/tests/functional/config/test_config_with_metadata.py index 2ccb47ddf532..0a0f7f975e29 100644 --- a/ydb/tests/functional/config/test_config_with_metadata.py +++ b/ydb/tests/functional/config/test_config_with_metadata.py @@ -177,7 +177,7 @@ def test_config_stored_in_config_store(self): replace_config_response = self.config_client.replace_config(config_yaml) assert_that(replace_config_response.operation.status == StatusIds.SUCCESS) - fetch_config_response = self.config_client.fetch_config() + fetch_config_response = self.config_client.fetch_all_configs() assert_that(fetch_config_response.operation.status == StatusIds.SUCCESS) result = config.FetchConfigResult() diff --git a/ydb/tests/library/clients/kikimr_config_client.py b/ydb/tests/library/clients/kikimr_config_client.py index f1b0af7c2984..44e657b47c8d 100644 --- a/ydb/tests/library/clients/kikimr_config_client.py +++ b/ydb/tests/library/clients/kikimr_config_client.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- import time import os +from enum import Enum import grpc @@ -25,6 +26,11 @@ def channels_list(): class ConfigClient(object): + class FetchTransform(Enum): + NONE = 1 + DETACH_STORAGE_CONFIG_SECTION = 2 + ATTACH_STORAGE_CONFIG_SECTION = 3 + def __init__(self, server, port, cluster=None, retry_count=1): self.server = server self.port = port @@ -56,18 +62,22 @@ def invoke(self, request, method): time.sleep(self.__retry_sleep_seconds) - def replace_config(self, yaml_config, storage_yaml_config=None): + def replace_config(self, main_config): request = config_api.ReplaceConfigRequest() - request.replace = yaml_config - if storage_yaml_config is not None: - request.storage_yaml_config = storage_yaml_config + request.replace = main_config return self.invoke(request, 'ReplaceConfig') - def fetch_config(self, dedicated_storage_section=False, dedicated_cluster_section=False): + def fetch_all_configs(self, transform=None): request = config_api.FetchConfigRequest() - # FIXME - # request.dedicated_storage_section = dedicated_storage_section - # request.dedicated_cluster_section = dedicated_cluster_section + settings = config_api.FetchConfigRequest.FetchModeAll() + + if transform == ConfigClient.FetchTransform.DETACH_STORAGE_CONFIG_SECTION: + settings.set_detach_storage_config_section() + elif transform == ConfigClient.FetchTransform.ATTACH_STORAGE_CONFIG_SECTION: + settings.set_attach_storage_config_section() + + request.all.CopyFrom(settings) + return self.invoke(request, 'FetchConfig') def bootstrap_cluster(self, self_assembly_uuid): From c5778c2b2ddf11c69499c8ed79ebe3474b61c3a6 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Mon, 17 Feb 2025 14:26:30 +0000 Subject: [PATCH 30/50] WIP --- .../sdk/cpp/client/ydb_config/ydb_config.cpp | 27 ++++++++++--- .../sdk/cpp/client/ydb_config/ydb_config.h | 38 ++++++++++++++++--- 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp index 63a95c1a324c..db577db02fb7 100644 --- a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp +++ b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp @@ -14,24 +14,27 @@ class TConfigClient::TImpl : public TClientImplCommon { TAsyncStatus ReplaceConfig(const TString& config) { auto request = MakeRequest(); - request.set_yaml_config(config); + request.set_replace(config); return RunSimple( std::move(request), &Ydb::Config::V1::BSConfigService::Stub::AsyncReplaceConfig); } + // FIXME add all other calls + TAsyncFetchConfigResult FetchConfig(const TConfigSettings& settings = {}) { auto request = MakeOperationRequest(settings); auto promise = NThreading::NewPromise(); auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { - TString config; + std::vector configs; if (Ydb::Config::FetchConfigResult result; any && any->UnpackTo(&result)) { - config = result.yaml_config(); + // FIXME + Y_UNUSED(result); } - TFetchConfigResult val(TStatus(std::move(status)), std::move(config)); + TFetchConfigResult val(TStatus(std::move(status)), std::move(configs)); promise.SetValue(std::move(val)); }; @@ -61,8 +64,20 @@ TConfigClient::TConfigClient(const TDriver& driver, const TCommonClientSettings& TConfigClient::~TConfigClient() = default; -TAsyncStatus TConfigClient::ReplaceConfig(const TString& config) { - return Impl_->ReplaceConfig(config); +TAsyncStatus TConfigClient::ReplaceConfig(const TString& mainConfig) { + return Impl_->ReplaceConfig(mainConfig); +} + +TAsyncStatus TConfigClient::ReplaceConfigDisableDedicatedStorageSection(const TString& mainConfig) { + return Impl_->ReplaceConfigDisableDedicatedStorageSection(mainConfig); +} + +TAsyncStatus TConfigClient::ReplaceConfigEnableDedicatedStorageSection(const TString& mainConfig, const TString& storageConfig) { + return Impl_->ReplaceConfigEnableDedicatedStorageSection(mainConfig, storageConfig); +} + +TAsyncStatus TConfigClient::ReplaceConfigWithDedicatedStorageSection(const TString& mainConfig, const TString& storageConfig) { + return Impl_->ReplaceConfigWithDedicatedStorageSection(mainConfig, storageConfig); } TAsyncFetchConfigResult TConfigClient::FetchConfig(const TConfigSettings& settings) { diff --git a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h index 3eacad66d1c3..1bd87e6cc821 100644 --- a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h +++ b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h @@ -10,23 +10,42 @@ #include #include +#include +#include namespace NYdb::NConfig { +struct TMainConfigMetadata { + +}; + +struct TStorageConfigMetadata { + +}; + +struct TDatabaseConfigMetadata { + +}; + +struct TConfig { + std::variant Metadata; + TString Config; +}; + struct TFetchConfigResult : public TStatus { TFetchConfigResult( TStatus&& status, - TString&& config) + std::vector&& configs) : TStatus(std::move(status)) , Config_(std::move(config)) {} - const TString& GetConfig() const { - return Config_; + const std::vector& GetConfigs() const { + return Configs_; } private: - TString Config_; + std::vector Configs_; }; using TAsyncFetchConfigResult = NThreading::TFuture; @@ -41,7 +60,16 @@ class TConfigClient { ~TConfigClient(); // Replace config - TAsyncStatus ReplaceConfig(const TString& config); + TAsyncStatus ReplaceConfig(const TString& mainConfig); + + // Replace config + TAsyncStatus ReplaceConfigDisableDedicatedStorageSection(const TString& mainConfig); + + // Replace config + TAsyncStatus ReplaceConfigEnableDedicatedStorageSection(const TString& mainConfig, const TString& storageConfig); + + // Replace config + TAsyncStatus ReplaceConfigWithDedicatedStorageSection(const TString& mainConfig, const TString& storageConfig); // Fetch current cluster storage config TAsyncFetchConfigResult FetchConfig(const TConfigSettings& settings = {}); From d432002780d58f29e19c19989ef307494b778c3b Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Mon, 17 Feb 2025 14:36:13 +0000 Subject: [PATCH 31/50] WIP --- .../sdk/cpp/client/ydb_config/ydb_config.cpp | 41 ++++++++++++++++--- .../sdk/cpp/client/ydb_config/ydb_config.h | 6 +-- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp index db577db02fb7..80fb14683658 100644 --- a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp +++ b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp @@ -12,7 +12,7 @@ class TConfigClient::TImpl : public TClientImplCommon { { } - TAsyncStatus ReplaceConfig(const TString& config) { + TAsyncStatus ReplaceConfig(const TString& mainConfig) { auto request = MakeRequest(); request.set_replace(config); @@ -21,7 +21,36 @@ class TConfigClient::TImpl : public TClientImplCommon { &Ydb::Config::V1::BSConfigService::Stub::AsyncReplaceConfig); } - // FIXME add all other calls + TAsyncStatus ReplaceConfig(const TString& mainConfig, const TString& storageConfig) { + auto request = MakeRequest(); + auto& replace = *request.mutable_replace_with_dedicated_storage_section(); + replace.set_main_config(mainConfig); + replace.set_storage_config(storageConfig); + + return RunSimple( + std::move(request), + &Ydb::Config::V1::BSConfigService::Stub::AsyncReplaceConfig); + } + + TAsyncStatus ReplaceConfigDisableDedicatedStorageSection(const TString& mainConfig) { + auto request = MakeRequest(); + request.set_replace_disable_dedicated_storage_section(config); + + return RunSimple( + std::move(request), + &Ydb::Config::V1::BSConfigService::Stub::AsyncReplaceConfig); + } + + TAsyncStatus ReplaceConfigEnableDedicatedStorageSection(const TString& mainConfig, const TString& storageConfig) { + auto request = MakeRequest(); + auto& replace = *request.mutable_replace_enable_dedicated_storage_section(); + replace.set_main_config(mainConfig); + replace.set_storage_config(storageConfig); + + return RunSimple( + std::move(request), + &Ydb::Config::V1::BSConfigService::Stub::AsyncReplaceConfig); + } TAsyncFetchConfigResult FetchConfig(const TConfigSettings& settings = {}) { auto request = MakeOperationRequest(settings); @@ -68,6 +97,10 @@ TAsyncStatus TConfigClient::ReplaceConfig(const TString& mainConfig) { return Impl_->ReplaceConfig(mainConfig); } +TAsyncStatus TConfigClient::ReplaceConfig(const TString& mainConfig, const TString& storageConfig) { + return Impl_->ReplaceConfig(mainConfig, storageConfig); +} + TAsyncStatus TConfigClient::ReplaceConfigDisableDedicatedStorageSection(const TString& mainConfig) { return Impl_->ReplaceConfigDisableDedicatedStorageSection(mainConfig); } @@ -76,10 +109,6 @@ TAsyncStatus TConfigClient::ReplaceConfigEnableDedicatedStorageSection(const TSt return Impl_->ReplaceConfigEnableDedicatedStorageSection(mainConfig, storageConfig); } -TAsyncStatus TConfigClient::ReplaceConfigWithDedicatedStorageSection(const TString& mainConfig, const TString& storageConfig) { - return Impl_->ReplaceConfigWithDedicatedStorageSection(mainConfig, storageConfig); -} - TAsyncFetchConfigResult TConfigClient::FetchConfig(const TConfigSettings& settings) { return Impl_->FetchConfig(settings); } diff --git a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h index 1bd87e6cc821..d72d922b1d83 100644 --- a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h +++ b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h @@ -63,13 +63,13 @@ class TConfigClient { TAsyncStatus ReplaceConfig(const TString& mainConfig); // Replace config - TAsyncStatus ReplaceConfigDisableDedicatedStorageSection(const TString& mainConfig); + TAsyncStatus ReplaceConfig(const TString& mainConfig, const TString& storageConfig); // Replace config - TAsyncStatus ReplaceConfigEnableDedicatedStorageSection(const TString& mainConfig, const TString& storageConfig); + TAsyncStatus ReplaceConfigDisableDedicatedStorageSection(const TString& mainConfig); // Replace config - TAsyncStatus ReplaceConfigWithDedicatedStorageSection(const TString& mainConfig, const TString& storageConfig); + TAsyncStatus ReplaceConfigEnableDedicatedStorageSection(const TString& mainConfig, const TString& storageConfig); // Fetch current cluster storage config TAsyncFetchConfigResult FetchConfig(const TConfigSettings& settings = {}); From d0b2bc05be0e0a681e42ba69336adab91adebac6 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Mon, 17 Feb 2025 14:48:10 +0000 Subject: [PATCH 32/50] WIP --- .../sdk/cpp/client/ydb_config/ydb_config.cpp | 30 +++++++++++++++++-- .../sdk/cpp/client/ydb_config/ydb_config.h | 15 +++++++--- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp index 80fb14683658..12a2790019c5 100644 --- a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp +++ b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp @@ -59,8 +59,34 @@ class TConfigClient::TImpl : public TClientImplCommon { auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { std::vector configs; if (Ydb::Config::FetchConfigResult result; any && any->UnpackTo(&result)) { - // FIXME - Y_UNUSED(result); + for (const auto& entry : result.config()) { + TKnownIdentityTypes identity; + + switch (entry.identity().type_case()) { + case Ydb::Config::ConfigIdentity::TypeCase::kMain: + identity = TMainConfigIdentity { + + }; + break; + case Ydb::Config::ConfigConfigIdentity::TypeCase::kStorage: + identity = TStorageConfigIdentity { + + }; + break; + case Ydb::Config::ConfigConfigIdentity::TypeCase::kDatabase: + identity = TDatabaseConfigIdentity { + + }; + break; + case Ydb::Config::ConfigIdentity::TypeCase::ACTION_NOT_SET: + break; // leave in monostate; uknown identity + } + + configs.push_back(TConfig{ + .Identity = std::monostate, + .Config = identity, + }); + } } TFetchConfigResult val(TStatus(std::move(status)), std::move(configs)); diff --git a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h index d72d922b1d83..5e4ef556929d 100644 --- a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h +++ b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h @@ -15,20 +15,27 @@ namespace NYdb::NConfig { -struct TMainConfigMetadata { +struct TMainConfigIdentity { }; -struct TStorageConfigMetadata { +struct TStorageConfigIdentity { }; -struct TDatabaseConfigMetadata { +struct TDatabaseConfigIdentity { }; +using TKnownIdentitiyTypes = std::variant< + std::monostate + , TMainConfigMetadata + , TStorageConfigMetadata + , TDatabaseConfigMetadata + >; + struct TConfig { - std::variant Metadata; + TKnownIdentityTypes Identity; TString Config; }; From f5623cdc79ad5a1942baa89e4a4d997df126d35d Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Mon, 17 Feb 2025 14:54:43 +0000 Subject: [PATCH 33/50] WIP --- ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp | 10 +++++++--- ydb/public/sdk/cpp/client/ydb_config/ydb_config.h | 10 +++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp index 12a2790019c5..c6a9907f6cdf 100644 --- a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp +++ b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp @@ -65,17 +65,21 @@ class TConfigClient::TImpl : public TClientImplCommon { switch (entry.identity().type_case()) { case Ydb::Config::ConfigIdentity::TypeCase::kMain: identity = TMainConfigIdentity { - + .Version = entry.identity().version(); + .Cluster = entry.identity().cluster(); }; break; case Ydb::Config::ConfigConfigIdentity::TypeCase::kStorage: identity = TStorageConfigIdentity { - + .Version = entry.identity().version(); + .Cluster = entry.identity().cluster(); }; break; case Ydb::Config::ConfigConfigIdentity::TypeCase::kDatabase: identity = TDatabaseConfigIdentity { - + .Version = entry.identity().version(); + .Cluster = entry.identity().cluster(); + .Database = entry.identity().database().database(); }; break; case Ydb::Config::ConfigIdentity::TypeCase::ACTION_NOT_SET: diff --git a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h index 5e4ef556929d..f54a45cc2ec8 100644 --- a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h +++ b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h @@ -16,15 +16,19 @@ namespace NYdb::NConfig { struct TMainConfigIdentity { - + ui64 Version; + TString Cluster; }; struct TStorageConfigIdentity { - + ui64 Version; + TString Cluster; }; struct TDatabaseConfigIdentity { - + ui64 Version; + TString Cluster; + TString Database; }; using TKnownIdentitiyTypes = std::variant< From 2cdcd645dd0ab30be827f10b1242b38c112c4975 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Mon, 17 Feb 2025 15:00:14 +0000 Subject: [PATCH 34/50] WIP --- ydb/public/sdk/cpp/client/ydb_config/ydb_config.h | 1 + 1 file changed, 1 insertion(+) diff --git a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h index f54a45cc2ec8..197195aae44d 100644 --- a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h +++ b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h @@ -61,6 +61,7 @@ struct TFetchConfigResult : public TStatus { using TAsyncFetchConfigResult = NThreading::TFuture; +// FIXME add settings for fetch and replace struct TConfigSettings : public NYdb::TOperationRequestSettings {}; class TConfigClient { From 5788a94ff63187c2fcf4c5850eca7333b4e44454 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Mon, 17 Feb 2025 15:14:53 +0000 Subject: [PATCH 35/50] WIP --- ydb/public/api/protos/ydb_config.proto | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/ydb/public/api/protos/ydb_config.proto b/ydb/public/api/protos/ydb_config.proto index 063333ff2022..3bef77fda109 100644 --- a/ydb/public/api/protos/ydb_config.proto +++ b/ydb/public/api/protos/ydb_config.proto @@ -10,10 +10,24 @@ option java_multiple_files = true; import "ydb/public/api/protos/ydb_operation.proto"; import "google/protobuf/empty.proto"; +// Main config is "the" main config used for configuring everyting +// For most installations this config is enough, because it +// allows database administrator to configure all entites: +// whole cluster, database, specific node(s), etc. message MainConfigIdentity {} +// Storage config allows administrator separate management of +// storage/infrastructure part from main/application part +// it is useful on large clusters, where enumeration of all +// physical machines takes too much space and make it inconvenient +// to store and manage this configuration data together with other message StorageConfigIdentity {} +// Database config allows to manage specific parts of configuration +// by database administrators themself. It mostly applies to +// feature flags enabled on particular database +// This type of config is disabled by default and can be enabled +// in configuration message DatabaseConfigIdentity { string database = 1; } @@ -107,10 +121,14 @@ message FetchConfigRequest { // Fetch will return MainConfig and StorageConfig derived from MainConfig // by detaching storage config + // (MainConfig) -> (MainConfig*, StorageConfig) + // MainConfig with asterisk means MainConfig with excluded storage-related sections google.protobuf.Empty detach_storage_config_section = 2; // Fetch will return MainConfig derived from MainConfig and StorageConfig // by attaching storage config to main config + // (MainConfig*, StorageConfig) -> (MainConfig) + // MainConfig with asterisk means MainConfig with excluded storage-related sections google.protobuf.Empty attach_storage_config_section = 3; } } From 1c75f867cad1426a54131043461d14e21e4d031b Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Mon, 17 Feb 2025 15:58:35 +0000 Subject: [PATCH 36/50] WIP --- .../sdk/cpp/client/ydb_config/ydb_config.cpp | 64 +++++++++---- .../sdk/cpp/client/ydb_config/ydb_config.h | 37 +++++-- .../ydb-cpp-sdk/client/config/config.h | 96 +++++++++++++------ 3 files changed, 141 insertions(+), 56 deletions(-) diff --git a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp index c6a9907f6cdf..e3724ca5b1bc 100644 --- a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp +++ b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp @@ -12,47 +12,55 @@ class TConfigClient::TImpl : public TClientImplCommon { { } - TAsyncStatus ReplaceConfig(const TString& mainConfig) { + TAsyncStatus ReplaceConfig(const TString& mainConfig, const TReplaceConfigSettings& settings = {}) { auto request = MakeRequest(); request.set_replace(config); + ApplyReplaceSettings(request, settings); + return RunSimple( std::move(request), &Ydb::Config::V1::BSConfigService::Stub::AsyncReplaceConfig); } - TAsyncStatus ReplaceConfig(const TString& mainConfig, const TString& storageConfig) { + TAsyncStatus ReplaceConfig(const TString& mainConfig, const TString& storageConfig, const TReplaceConfigSettings& settings = {}) { auto request = MakeRequest(); auto& replace = *request.mutable_replace_with_dedicated_storage_section(); replace.set_main_config(mainConfig); replace.set_storage_config(storageConfig); + ApplyReplaceSettings(request, settings); + return RunSimple( std::move(request), &Ydb::Config::V1::BSConfigService::Stub::AsyncReplaceConfig); } - TAsyncStatus ReplaceConfigDisableDedicatedStorageSection(const TString& mainConfig) { + TAsyncStatus ReplaceConfigDisableDedicatedStorageSection(const TString& mainConfig, const TReplaceConfigSettings& settings = {}) { auto request = MakeRequest(); request.set_replace_disable_dedicated_storage_section(config); + ApplyReplaceSettings(request, settings); + return RunSimple( std::move(request), &Ydb::Config::V1::BSConfigService::Stub::AsyncReplaceConfig); } - TAsyncStatus ReplaceConfigEnableDedicatedStorageSection(const TString& mainConfig, const TString& storageConfig) { + TAsyncStatus ReplaceConfigEnableDedicatedStorageSection(const TString& mainConfig, const TString& storageConfig, const TReplaceConfigSettings& settings = {}) { auto request = MakeRequest(); auto& replace = *request.mutable_replace_enable_dedicated_storage_section(); replace.set_main_config(mainConfig); replace.set_storage_config(storageConfig); + ApplyReplaceSettings(request, settings); + return RunSimple( std::move(request), &Ydb::Config::V1::BSConfigService::Stub::AsyncReplaceConfig); } - TAsyncFetchConfigResult FetchConfig(const TConfigSettings& settings = {}) { + TAsyncFetchConfigResult FetchConfig(const TFetchConfigSettings& settings = {}) { auto request = MakeOperationRequest(settings); auto promise = NThreading::NewPromise(); @@ -107,7 +115,8 @@ class TConfigClient::TImpl : public TClientImplCommon { return promise.GetFuture(); } - TAsyncStatus BootstrapCluster(const TString& selfAssemblyUUID) { + TAsyncStatus BootstrapCluster(const TString& selfAssemblyUUID, const TBootstrapClusterSettings& settings = {}) { + Y_UNUSED(settings); auto request = MakeRequest(); request.set_self_assembly_uuid(selfAssemblyUUID); @@ -115,6 +124,12 @@ class TConfigClient::TImpl : public TClientImplCommon { Ydb::Config::BootstrapClusterResponse>(std::move(request), &Ydb::Config::V1::BSConfigService::Stub::AsyncBootstrapCluster); } +private: + static void ApplyReplaceSettings(auto& request, const TReplaceConfigSettings& settings) const { + request.set_dry_run(settings.DryRun_); + request.set_allow_unknown_fields(settings.AllowUnknownFields_); + request.set_bypass_checks(settings.BypassChecks_); + } }; TConfigClient::TConfigClient(const TDriver& driver, const TCommonClientSettings& settings) @@ -123,28 +138,45 @@ TConfigClient::TConfigClient(const TDriver& driver, const TCommonClientSettings& TConfigClient::~TConfigClient() = default; -TAsyncStatus TConfigClient::ReplaceConfig(const TString& mainConfig) { - return Impl_->ReplaceConfig(mainConfig); +TAsyncStatus TConfigClient::ReplaceConfig( + const TString& mainConfig, + const TReplaceConfigSettings& settings) +{ + return Impl_->ReplaceConfig(mainConfig, settings); } -TAsyncStatus TConfigClient::ReplaceConfig(const TString& mainConfig, const TString& storageConfig) { - return Impl_->ReplaceConfig(mainConfig, storageConfig); +TAsyncStatus TConfigClient::ReplaceConfig( + const TString& mainConfig, + const TString& storageConfig, + const TReplaceConfigSettings& settings) +{ + return Impl_->ReplaceConfig(mainConfig, storageConfig, settings); } -TAsyncStatus TConfigClient::ReplaceConfigDisableDedicatedStorageSection(const TString& mainConfig) { - return Impl_->ReplaceConfigDisableDedicatedStorageSection(mainConfig); +TAsyncStatus TConfigClient::ReplaceConfigDisableDedicatedStorageSection( + const TString& mainConfig, + const TReplaceConfigSettings& settings) +{ + return Impl_->ReplaceConfigDisableDedicatedStorageSection(mainConfig, settings); } -TAsyncStatus TConfigClient::ReplaceConfigEnableDedicatedStorageSection(const TString& mainConfig, const TString& storageConfig) { +TAsyncStatus TConfigClient::ReplaceConfigEnableDedicatedStorageSection( + const TString& mainConfig, + const TString& storageConfig, + const TReplaceConfigSettings& settings) +{ return Impl_->ReplaceConfigEnableDedicatedStorageSection(mainConfig, storageConfig); } -TAsyncFetchConfigResult TConfigClient::FetchConfig(const TConfigSettings& settings) { +TAsyncFetchConfigResult TConfigClient::FetchConfig(const TFetchConfigSettings& settings) { return Impl_->FetchConfig(settings); } -TAsyncStatus TConfigClient::BootstrapCluster(const TString& selfAssemblyUUID) { - return Impl_->BootstrapCluster(selfAssemblyUUID); +TAsyncStatus TConfigClient::BootstrapCluster( + const TString& selfAssemblyUUID, + const TBootstrapClusterSettings& settings) +{ + return Impl_->BootstrapCluster(selfAssemblyUUID, settings); } } diff --git a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h index 197195aae44d..a53279643676 100644 --- a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h +++ b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h @@ -15,6 +15,16 @@ namespace NYdb::NConfig { +struct TReplaceConfigSettings : public NYdb::TOperationRequestSettings { + FLUENT_SETTING_FLAG(DryRun); + FLUENT_SETTING_FLAG(AllowUnknownFields); + FLUENT_SETTING_FLAG(BypassChecks); +}; + +struct TFetchConfigSettings : public NYdb::TOperationRequestSettings {}; + +struct TBootstrapClusterSettings : public NYdb::TOperationRequestSettings {}; + struct TMainConfigIdentity { ui64 Version; TString Cluster; @@ -61,9 +71,6 @@ struct TFetchConfigResult : public TStatus { using TAsyncFetchConfigResult = NThreading::TFuture; -// FIXME add settings for fetch and replace -struct TConfigSettings : public NYdb::TOperationRequestSettings {}; - class TConfigClient { public: @@ -72,22 +79,34 @@ class TConfigClient { ~TConfigClient(); // Replace config - TAsyncStatus ReplaceConfig(const TString& mainConfig); + TAsyncStatus ReplaceConfig( + const TString& mainConfig, + const TReplaceConfigSettings& settings = {}); // Replace config - TAsyncStatus ReplaceConfig(const TString& mainConfig, const TString& storageConfig); + TAsyncStatus ReplaceConfig( + const TString& mainConfig, + const TString& storageConfig, + const TReplaceConfigSettings& settings = {}); // Replace config - TAsyncStatus ReplaceConfigDisableDedicatedStorageSection(const TString& mainConfig); + TAsyncStatus ReplaceConfigDisableDedicatedStorageSection( + const TString& mainConfig, + const TReplaceConfigSettings& settings = {}); // Replace config - TAsyncStatus ReplaceConfigEnableDedicatedStorageSection(const TString& mainConfig, const TString& storageConfig); + TAsyncStatus ReplaceConfigEnableDedicatedStorageSection( + const TString& mainConfig, + const TString& storageConfig, + const TReplaceConfigSettings& settings = {}); // Fetch current cluster storage config - TAsyncFetchConfigResult FetchConfig(const TConfigSettings& settings = {}); + TAsyncFetchConfigResult FetchConfig(const TFetchConfigSettings& settings = {}); // Bootstrap cluster with automatic configuration - TAsyncStatus BootstrapCluster(const TString& selfAssemblyUUID); + TAsyncStatus BootstrapCluster( + const TString& selfAssemblyUUID, + const TBootstrapClusterSettings& settings = {}); private: class TImpl; diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/config/config.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/config/config.h index f829af09c5a6..a27f7a7b42cc 100644 --- a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/config/config.h +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/config/config.h @@ -11,45 +11,62 @@ namespace NYdb::inline V3::NConfig { +struct TReplaceConfigSettings : public NYdb::TOperationRequestSettings { + FLUENT_SETTING_FLAG(DryRun); + FLUENT_SETTING_FLAG(AllowUnknownFields); + FLUENT_SETTING_FLAG(BypassChecks); +}; + +struct TFetchConfigSettings : public NYdb::TOperationRequestSettings {}; + +struct TBootstrapClusterSettings : public NYdb::TOperationRequestSettings {}; + +struct TMainConfigIdentity { + ui64 Version; + TString Cluster; +}; + +struct TStorageConfigIdentity { + ui64 Version; + TString Cluster; +}; + +struct TDatabaseConfigIdentity { + ui64 Version; + TString Cluster; + TString Database; +}; + +using TKnownIdentitiyTypes = std::variant< + std::monostate + , TMainConfigMetadata + , TStorageConfigMetadata + , TDatabaseConfigMetadata + >; + +struct TConfig { + TKnownIdentityTypes Identity; + TString Config; +}; + struct TFetchConfigResult : public TStatus { TFetchConfigResult( TStatus&& status, - std::string&& main_config, - std::string&& storage_config) + std::vector&& configs) : TStatus(std::move(status)) - , MainConfig_(std::move(main_config)) - , StorageConfig_(std::move(storage_config)) + , Config_(std::move(config)) {} - const std::string& GetMainConfig() const { - return MainConfig_; - } - - const std::string& GetStorageConfig() const { - return StorageConfig_; + const std::vector& GetConfigs() const { + return Configs_; } private: - std::string MainConfig_; - std::string StorageConfig_; + std::vector Configs_; }; using TAsyncFetchConfigResult = NThreading::TFuture; -struct TReplaceConfigSettings : public NYdb::TOperationRequestSettings { - FLUENT_SETTING_OPTIONAL(bool, SwitchDedicatedStorageSection); - FLUENT_SETTING_FLAG(DedicatedConfigMode); - FLUENT_SETTING_FLAG(DryRun); - FLUENT_SETTING_FLAG(AllowUnknownFields); - FLUENT_SETTING_FLAG(AllowAbsentDatabase); - FLUENT_SETTING_FLAG(AllowIncorrectVersion); - FLUENT_SETTING_FLAG(AllowIncorrectCluster); -}; - -struct TFetchConfigSettings : public NYdb::TOperationRequestSettings {}; - -struct TBootstrapClusterSettings : public NYdb::TOperationRequestSettings {}; - class TConfigClient { public: explicit TConfigClient(const TDriver& driver, const TCommonClientSettings& settings = {}); @@ -58,16 +75,33 @@ class TConfigClient { // Replace config TAsyncStatus ReplaceConfig( - const std::optional& main_config, - const std::optional& storage_config, + const TString& mainConfig, + const TReplaceConfigSettings& settings = {}); + + // Replace config + TAsyncStatus ReplaceConfig( + const TString& mainConfig, + const TString& storageConfig, + const TReplaceConfigSettings& settings = {}); + + // Replace config + TAsyncStatus ReplaceConfigDisableDedicatedStorageSection( + const TString& mainConfig, + const TReplaceConfigSettings& settings = {}); + + // Replace config + TAsyncStatus ReplaceConfigEnableDedicatedStorageSection( + const TString& mainConfig, + const TString& storageConfig, const TReplaceConfigSettings& settings = {}); // Fetch current cluster storage config - TAsyncFetchConfigResult FetchConfig(bool dedicated_storage_section, bool dedicated_cluster_section, - const TFetchConfigSettings& settings = {}); + TAsyncFetchConfigResult FetchConfig(const TFetchConfigSettings& settings = {}); // Bootstrap cluster with automatic configuration - TAsyncStatus BootstrapCluster(const std::string& selfAssemblyUUID, const TBootstrapClusterSettings& settings = {}); + TAsyncStatus BootstrapCluster( + const TString& selfAssemblyUUID, + const TBootstrapClusterSettings& settings = {}); private: class TImpl; From c37c082f89db2f99e8f5408b8bde2191f9fa8b2a Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Mon, 17 Feb 2025 16:04:49 +0000 Subject: [PATCH 37/50] WIP --- .../sdk/cpp/src/client/config/config.cpp | 185 ++++++++++++------ 1 file changed, 123 insertions(+), 62 deletions(-) diff --git a/ydb/public/sdk/cpp/src/client/config/config.cpp b/ydb/public/sdk/cpp/src/client/config/config.cpp index a012612e0133..1aba25bd5ba8 100644 --- a/ydb/public/sdk/cpp/src/client/config/config.cpp +++ b/ydb/public/sdk/cpp/src/client/config/config.cpp @@ -14,84 +14,123 @@ class TConfigClient::TImpl : public TClientImplCommon { { } - TAsyncStatus ReplaceConfig( - const std::optional& main_config, - const std::optional& storage_config, - const TReplaceConfigSettings& settings) - { + TAsyncStatus ReplaceConfig(const TString& mainConfig, const TReplaceConfigSettings& settings = {}) { auto request = MakeRequest(); + request.set_replace(config); + + ApplyReplaceSettings(request, settings); + + return RunSimple( + std::move(request), + &Ydb::Config::V1::BSConfigService::Stub::AsyncReplaceConfig); + } - Y_UNUSED(main_config, storage_config, settings); // FIXME + TAsyncStatus ReplaceConfig(const TString& mainConfig, const TString& storageConfig, const TReplaceConfigSettings& settings = {}) { + auto request = MakeRequest(); + auto& replace = *request.mutable_replace_with_dedicated_storage_section(); + replace.set_main_config(mainConfig); + replace.set_storage_config(storageConfig); - // if (yaml_config) { - // request.set_main_config(*yaml_config); - // } + ApplyReplaceSettings(request, settings); - // if (storage_yaml_config) { - // request.set_storage_config(*storage_yaml_config); - // } + return RunSimple( + std::move(request), + &Ydb::Config::V1::BSConfigService::Stub::AsyncReplaceConfig); + } - // if (settings.SwitchDedicatedStorageSection_) { - // request.set_switch_dedicated_storage_section(*settings.SwitchDedicatedStorageSection_); - // } + TAsyncStatus ReplaceConfigDisableDedicatedStorageSection(const TString& mainConfig, const TReplaceConfigSettings& settings = {}) { + auto request = MakeRequest(); + request.set_replace_disable_dedicated_storage_section(config); - // request.set_dedicated_config_mode(settings.DedicatedConfigMode_); - // request.set_dry_run(settings.DryRun_); - // request.set_allow_unknown_fields(settings.AllowUnknownFields_); - // request.set_allow_absent_database(settings.AllowAbsentDatabase_); - // request.set_allow_incorrect_version(settings.AllowIncorrectVersion_); - // request.set_allow_incorrect_cluster(settings.AllowIncorrectCluster_); + ApplyReplaceSettings(request, settings); - return RunSimple( + return RunSimple( std::move(request), - &Ydb::Config::V1::ConfigService::Stub::AsyncReplaceConfig, - TRpcRequestSettings::Make(settings)); + &Ydb::Config::V1::BSConfigService::Stub::AsyncReplaceConfig); } - TAsyncFetchConfigResult FetchConfig(bool dedicated_storage_section, bool dedicated_cluster_section, - const TFetchConfigSettings& settings) - { + TAsyncStatus ReplaceConfigEnableDedicatedStorageSection(const TString& mainConfig, const TString& storageConfig, const TReplaceConfigSettings& settings = {}) { + auto request = MakeRequest(); + auto& replace = *request.mutable_replace_enable_dedicated_storage_section(); + replace.set_main_config(mainConfig); + replace.set_storage_config(storageConfig); + + ApplyReplaceSettings(request, settings); + + return RunSimple( + std::move(request), + &Ydb::Config::V1::BSConfigService::Stub::AsyncReplaceConfig); + } + + TAsyncFetchConfigResult FetchConfig(const TFetchConfigSettings& settings = {}) { auto request = MakeOperationRequest(settings); - Y_UNUSED(dedicated_storage_section, dedicated_cluster_section, settings); // FIXME - // if (dedicated_storage_section) { - // request.set_dedicated_storage_section(true); - // } - // if (dedicated_cluster_section) { - // request.set_dedicated_cluster_section(true); - // } auto promise = NThreading::NewPromise(); auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { - NYdb::TStringType config; - NYdb::TStringType storage_config; - Y_UNUSED(any); - // if (Ydb::Config::FetchConfigResult result; any && any->UnpackTo(&result)) { - // config = result.main_config(); - // storage_config = result.storage_config(); - // } - - TFetchConfigResult val(TStatus(std::move(status)), std::string{std::move(config)}, - std::string{std::move(storage_config)}); - promise.SetValue(std::move(val)); - }; - - Connections_->RunDeferred( + std::vector configs; + if (Ydb::Config::FetchConfigResult result; any && any->UnpackTo(&result)) { + for (const auto& entry : result.config()) { + TKnownIdentityTypes identity; + + switch (entry.identity().type_case()) { + case Ydb::Config::ConfigIdentity::TypeCase::kMain: + identity = TMainConfigIdentity { + .Version = entry.identity().version(); + .Cluster = entry.identity().cluster(); + }; + break; + case Ydb::Config::ConfigConfigIdentity::TypeCase::kStorage: + identity = TStorageConfigIdentity { + .Version = entry.identity().version(); + .Cluster = entry.identity().cluster(); + }; + break; + case Ydb::Config::ConfigConfigIdentity::TypeCase::kDatabase: + identity = TDatabaseConfigIdentity { + .Version = entry.identity().version(); + .Cluster = entry.identity().cluster(); + .Database = entry.identity().database().database(); + }; + break; + case Ydb::Config::ConfigIdentity::TypeCase::ACTION_NOT_SET: + break; // leave in monostate; uknown identity + } + + configs.push_back(TConfig{ + .Identity = std::monostate, + .Config = identity, + }); + } + } + + TFetchConfigResult val(TStatus(std::move(status)), std::move(configs)); + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred( std::move(request), extractor, - &Ydb::Config::V1::ConfigService::Stub::AsyncFetchConfig, + &Ydb::Config::V1::BSConfigService::Stub::AsyncFetchConfig, DbDriverState_, INITIAL_DEFERRED_CALL_DELAY, TRpcRequestSettings::Make(settings)); return promise.GetFuture(); } - TAsyncStatus BootstrapCluster(const std::string& selfAssemblyUUID, const TBootstrapClusterSettings& settings) { + TAsyncStatus BootstrapCluster(const TString& selfAssemblyUUID, const TBootstrapClusterSettings& settings = {}) { + Y_UNUSED(settings); auto request = MakeRequest(); request.set_self_assembly_uuid(selfAssemblyUUID); - return RunSimple(std::move(request), - &Ydb::Config::V1::ConfigService::Stub::AsyncBootstrapCluster, - TRpcRequestSettings::Make(settings)); + &Ydb::Config::V1::BSConfigService::Stub::AsyncBootstrapCluster); + } +private: + static void ApplyReplaceSettings(auto& request, const TReplaceConfigSettings& settings) const { + request.set_dry_run(settings.DryRun_); + request.set_allow_unknown_fields(settings.AllowUnknownFields_); + request.set_bypass_checks(settings.BypassChecks_); } }; @@ -102,22 +141,44 @@ TConfigClient::TConfigClient(const TDriver& driver, const TCommonClientSettings& TConfigClient::~TConfigClient() = default; TAsyncStatus TConfigClient::ReplaceConfig( - const std::optional& yaml_config, - const std::optional& storage_yaml_config, - const TReplaceConfigSettings& settings) + const TString& mainConfig, + const TReplaceConfigSettings& settings) { - return Impl_->ReplaceConfig(yaml_config, storage_yaml_config, settings); + return Impl_->ReplaceConfig(mainConfig, settings); } -TAsyncFetchConfigResult TConfigClient::FetchConfig(bool dedicated_storage_section, - bool dedicated_cluster_section, const TFetchConfigSettings& settings) +TAsyncStatus TConfigClient::ReplaceConfig( + const TString& mainConfig, + const TString& storageConfig, + const TReplaceConfigSettings& settings) { - return Impl_->FetchConfig(dedicated_storage_section, dedicated_cluster_section, settings); + return Impl_->ReplaceConfig(mainConfig, storageConfig, settings); } -TAsyncStatus TConfigClient::BootstrapCluster(const std::string& selfAssemblyUUID, const TBootstrapClusterSettings& settings) { - return Impl_->BootstrapCluster(selfAssemblyUUID, settings); +TAsyncStatus TConfigClient::ReplaceConfigDisableDedicatedStorageSection( + const TString& mainConfig, + const TReplaceConfigSettings& settings) +{ + return Impl_->ReplaceConfigDisableDedicatedStorageSection(mainConfig, settings); } +TAsyncStatus TConfigClient::ReplaceConfigEnableDedicatedStorageSection( + const TString& mainConfig, + const TString& storageConfig, + const TReplaceConfigSettings& settings) +{ + return Impl_->ReplaceConfigEnableDedicatedStorageSection(mainConfig, storageConfig); +} + +TAsyncFetchConfigResult TConfigClient::FetchConfig(const TFetchConfigSettings& settings) { + return Impl_->FetchConfig(settings); +} + +TAsyncStatus TConfigClient::BootstrapCluster( + const TString& selfAssemblyUUID, + const TBootstrapClusterSettings& settings) +{ + return Impl_->BootstrapCluster(selfAssemblyUUID, settings); +} } From 2e8492ff32e49aba06754fe855c84bcdd3f694b8 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Mon, 17 Feb 2025 16:32:28 +0000 Subject: [PATCH 38/50] WIP --- .../ydb_cli/commands/ydb_dynamic_config.cpp | 3 +- .../ydb_cli/commands/ydb_storage_config.cpp | 35 ++++++----- .../sdk/cpp/client/ydb_config/ydb_config.cpp | 2 +- .../sdk/cpp/client/ydb_config/ydb_config.h | 12 ++-- .../ydb-cpp-sdk/client/config/config.h | 12 ++-- .../sdk/cpp/src/client/config/config.cpp | 58 +++++++++---------- 6 files changed, 62 insertions(+), 60 deletions(-) diff --git a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp index 387ba381d221..8eed41bd9ad8 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp @@ -186,8 +186,7 @@ int TCommandConfigReplace::Run(TConfig& config) { NYdb::NConfig::TReplaceConfigSettings settings; if (Force) { - settings.AllowIncorrectVersion(); - settings.AllowIncorrectCluster(); + settings.BypassChecks(); } if (DryRun) { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp b/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp index 62c4bf9dba5e..6f93d6362755 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp @@ -63,11 +63,13 @@ void TCommandStorageConfigFetch::Parse(TConfig& config) { int TCommandStorageConfigFetch::Run(TConfig& config) { auto driver = std::make_unique(CreateDriver(config)); auto client = NYdb::NConfig::TConfigClient(*driver); - auto result = client.FetchConfig(DedicatedStorageSection, DedicatedClusterSection).GetValueSync(); + // FIXME + // auto result = client.FetchConfig(DedicatedStorageSection, DedicatedClusterSection).GetValueSync(); + auto result = client.FetchConfig().GetValueSync(); NStatusHelpers::ThrowOnError(result); - const auto& clusterConfig = result.GetMainConfig(); - const auto& storageConfig = result.GetStorageConfig(); + TString clusterConfig; // = result.GetMainConfig(); + TString storageConfig; // = result.GetStorageConfig(); if (!clusterConfig.empty()) { if (!storageConfig.empty() || DedicatedStorageSection) { @@ -140,19 +142,20 @@ void TCommandStorageConfigReplace::Parse(TConfig& config) { } int TCommandStorageConfigReplace::Run(TConfig& config) { - std::unique_ptr driver = std::make_unique(CreateDriver(config)); - auto client = NYdb::NConfig::TConfigClient(*driver); - NYdb::NConfig::TReplaceConfigSettings settings; - settings - .SwitchDedicatedStorageSection(SwitchDedicatedStorageSection) - .DedicatedConfigMode(DedicatedConfigMode); - - auto status = client.ReplaceConfig(ClusterYaml, StorageYaml, settings).GetValueSync(); - NStatusHelpers::ThrowOnError(status); - - if (!status.GetIssues()) { - Cout << status << Endl; - } + Y_UNUSED(config); // FIXME + // std::unique_ptr driver = std::make_unique(CreateDriver(config)); + // auto client = NYdb::NConfig::TConfigClient(*driver); + // NYdb::NConfig::TReplaceConfigSettings settings; + // settings + // .SwitchDedicatedStorageSection(SwitchDedicatedStorageSection) + // .DedicatedConfigMode(DedicatedConfigMode); + + // auto status = client.ReplaceConfig(ClusterYaml, StorageYaml, settings).GetValueSync(); + // NStatusHelpers::ThrowOnError(status); + + // if (!status.GetIssues()) { + // Cout << status << Endl; + // } return EXIT_SUCCESS; } diff --git a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp index e3724ca5b1bc..06d1ffc2163d 100644 --- a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp +++ b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp @@ -68,7 +68,7 @@ class TConfigClient::TImpl : public TClientImplCommon { std::vector configs; if (Ydb::Config::FetchConfigResult result; any && any->UnpackTo(&result)) { for (const auto& entry : result.config()) { - TKnownIdentityTypes identity; + TIdentityTypes identity; switch (entry.identity().type_case()) { case Ydb::Config::ConfigIdentity::TypeCase::kMain: diff --git a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h index a53279643676..35bc431eac5f 100644 --- a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h +++ b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h @@ -41,15 +41,15 @@ struct TDatabaseConfigIdentity { TString Database; }; -using TKnownIdentitiyTypes = std::variant< +using TKnownIdentityTypes = std::variant< std::monostate - , TMainConfigMetadata - , TStorageConfigMetadata - , TDatabaseConfigMetadata + , TMainConfigIdentity + , TStorageConfigIdentity + , TDatabaseConfigIdentity >; struct TConfig { - TKnownIdentityTypes Identity; + TIdentityTypes Identity; TString Config; }; @@ -58,7 +58,7 @@ struct TFetchConfigResult : public TStatus { TStatus&& status, std::vector&& configs) : TStatus(std::move(status)) - , Config_(std::move(config)) + , Configs_(std::move(configs)) {} const std::vector& GetConfigs() const { diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/config/config.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/config/config.h index a27f7a7b42cc..da7bfd51b5c6 100644 --- a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/config/config.h +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/config/config.h @@ -37,15 +37,15 @@ struct TDatabaseConfigIdentity { TString Database; }; -using TKnownIdentitiyTypes = std::variant< +using TIdentityTypes = std::variant< std::monostate - , TMainConfigMetadata - , TStorageConfigMetadata - , TDatabaseConfigMetadata + , TMainConfigIdentity + , TStorageConfigIdentity + , TDatabaseConfigIdentity >; struct TConfig { - TKnownIdentityTypes Identity; + TIdentityTypes Identity; TString Config; }; @@ -54,7 +54,7 @@ struct TFetchConfigResult : public TStatus { TStatus&& status, std::vector&& configs) : TStatus(std::move(status)) - , Config_(std::move(config)) + , Configs_(std::move(configs)) {} const std::vector& GetConfigs() const { diff --git a/ydb/public/sdk/cpp/src/client/config/config.cpp b/ydb/public/sdk/cpp/src/client/config/config.cpp index 1aba25bd5ba8..187ef46d1cc9 100644 --- a/ydb/public/sdk/cpp/src/client/config/config.cpp +++ b/ydb/public/sdk/cpp/src/client/config/config.cpp @@ -16,13 +16,13 @@ class TConfigClient::TImpl : public TClientImplCommon { TAsyncStatus ReplaceConfig(const TString& mainConfig, const TReplaceConfigSettings& settings = {}) { auto request = MakeRequest(); - request.set_replace(config); + request.set_replace(mainConfig); ApplyReplaceSettings(request, settings); - return RunSimple( + return RunSimple( std::move(request), - &Ydb::Config::V1::BSConfigService::Stub::AsyncReplaceConfig); + &Ydb::Config::V1::ConfigService::Stub::AsyncReplaceConfig); } TAsyncStatus ReplaceConfig(const TString& mainConfig, const TString& storageConfig, const TReplaceConfigSettings& settings = {}) { @@ -33,20 +33,20 @@ class TConfigClient::TImpl : public TClientImplCommon { ApplyReplaceSettings(request, settings); - return RunSimple( + return RunSimple( std::move(request), - &Ydb::Config::V1::BSConfigService::Stub::AsyncReplaceConfig); + &Ydb::Config::V1::ConfigService::Stub::AsyncReplaceConfig); } TAsyncStatus ReplaceConfigDisableDedicatedStorageSection(const TString& mainConfig, const TReplaceConfigSettings& settings = {}) { auto request = MakeRequest(); - request.set_replace_disable_dedicated_storage_section(config); + request.set_replace_disable_dedicated_storage_section(mainConfig); ApplyReplaceSettings(request, settings); - return RunSimple( + return RunSimple( std::move(request), - &Ydb::Config::V1::BSConfigService::Stub::AsyncReplaceConfig); + &Ydb::Config::V1::ConfigService::Stub::AsyncReplaceConfig); } TAsyncStatus ReplaceConfigEnableDedicatedStorageSection(const TString& mainConfig, const TString& storageConfig, const TReplaceConfigSettings& settings = {}) { @@ -57,9 +57,9 @@ class TConfigClient::TImpl : public TClientImplCommon { ApplyReplaceSettings(request, settings); - return RunSimple( + return RunSimple( std::move(request), - &Ydb::Config::V1::BSConfigService::Stub::AsyncReplaceConfig); + &Ydb::Config::V1::ConfigService::Stub::AsyncReplaceConfig); } TAsyncFetchConfigResult FetchConfig(const TFetchConfigSettings& settings = {}) { @@ -70,35 +70,35 @@ class TConfigClient::TImpl : public TClientImplCommon { std::vector configs; if (Ydb::Config::FetchConfigResult result; any && any->UnpackTo(&result)) { for (const auto& entry : result.config()) { - TKnownIdentityTypes identity; + TIdentityTypes identity; switch (entry.identity().type_case()) { case Ydb::Config::ConfigIdentity::TypeCase::kMain: identity = TMainConfigIdentity { - .Version = entry.identity().version(); - .Cluster = entry.identity().cluster(); + .Version = entry.identity().version(), + .Cluster = entry.identity().cluster(), }; break; - case Ydb::Config::ConfigConfigIdentity::TypeCase::kStorage: + case Ydb::Config::ConfigIdentity::TypeCase::kStorage: identity = TStorageConfigIdentity { - .Version = entry.identity().version(); - .Cluster = entry.identity().cluster(); + .Version = entry.identity().version(), + .Cluster = entry.identity().cluster(), }; break; - case Ydb::Config::ConfigConfigIdentity::TypeCase::kDatabase: + case Ydb::Config::ConfigIdentity::TypeCase::kDatabase: identity = TDatabaseConfigIdentity { - .Version = entry.identity().version(); - .Cluster = entry.identity().cluster(); - .Database = entry.identity().database().database(); + .Version = entry.identity().version(), + .Cluster = entry.identity().cluster(), + .Database = entry.identity().database().database(), }; break; - case Ydb::Config::ConfigIdentity::TypeCase::ACTION_NOT_SET: + case Ydb::Config::ConfigIdentity::TypeCase::TYPE_NOT_SET: break; // leave in monostate; uknown identity } configs.push_back(TConfig{ - .Identity = std::monostate, - .Config = identity, + .Identity = identity, + .Config = entry.config(), }); } } @@ -107,10 +107,10 @@ class TConfigClient::TImpl : public TClientImplCommon { promise.SetValue(std::move(val)); }; - Connections_->RunDeferred( + Connections_->RunDeferred( std::move(request), extractor, - &Ydb::Config::V1::BSConfigService::Stub::AsyncFetchConfig, + &Ydb::Config::V1::ConfigService::Stub::AsyncFetchConfig, DbDriverState_, INITIAL_DEFERRED_CALL_DELAY, TRpcRequestSettings::Make(settings)); @@ -122,12 +122,12 @@ class TConfigClient::TImpl : public TClientImplCommon { auto request = MakeRequest(); request.set_self_assembly_uuid(selfAssemblyUUID); - return RunSimple(std::move(request), - &Ydb::Config::V1::BSConfigService::Stub::AsyncBootstrapCluster); + &Ydb::Config::V1::ConfigService::Stub::AsyncBootstrapCluster); } private: - static void ApplyReplaceSettings(auto& request, const TReplaceConfigSettings& settings) const { + static void ApplyReplaceSettings(auto& request, const TReplaceConfigSettings& settings) { request.set_dry_run(settings.DryRun_); request.set_allow_unknown_fields(settings.AllowUnknownFields_); request.set_bypass_checks(settings.BypassChecks_); @@ -167,7 +167,7 @@ TAsyncStatus TConfigClient::ReplaceConfigEnableDedicatedStorageSection( const TString& storageConfig, const TReplaceConfigSettings& settings) { - return Impl_->ReplaceConfigEnableDedicatedStorageSection(mainConfig, storageConfig); + return Impl_->ReplaceConfigEnableDedicatedStorageSection(mainConfig, storageConfig, settings); } TAsyncFetchConfigResult TConfigClient::FetchConfig(const TFetchConfigSettings& settings) { From dd5d4de990272af3056b47c91eeba8b452e6c219 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Mon, 17 Feb 2025 16:46:25 +0000 Subject: [PATCH 39/50] WIP --- .../sdk/cpp/client/ydb_config/ydb_config.cpp | 56 +++++++++---------- .../sdk/cpp/client/ydb_config/ydb_config.h | 3 +- 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp index 06d1ffc2163d..fc41c73cf3d9 100644 --- a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp +++ b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp @@ -14,13 +14,13 @@ class TConfigClient::TImpl : public TClientImplCommon { TAsyncStatus ReplaceConfig(const TString& mainConfig, const TReplaceConfigSettings& settings = {}) { auto request = MakeRequest(); - request.set_replace(config); + request.set_replace(mainConfig); ApplyReplaceSettings(request, settings); - return RunSimple( + return RunSimple( std::move(request), - &Ydb::Config::V1::BSConfigService::Stub::AsyncReplaceConfig); + &Ydb::Config::V1::ConfigService::Stub::AsyncReplaceConfig); } TAsyncStatus ReplaceConfig(const TString& mainConfig, const TString& storageConfig, const TReplaceConfigSettings& settings = {}) { @@ -31,20 +31,20 @@ class TConfigClient::TImpl : public TClientImplCommon { ApplyReplaceSettings(request, settings); - return RunSimple( + return RunSimple( std::move(request), - &Ydb::Config::V1::BSConfigService::Stub::AsyncReplaceConfig); + &Ydb::Config::V1::ConfigService::Stub::AsyncReplaceConfig); } TAsyncStatus ReplaceConfigDisableDedicatedStorageSection(const TString& mainConfig, const TReplaceConfigSettings& settings = {}) { auto request = MakeRequest(); - request.set_replace_disable_dedicated_storage_section(config); + request.set_replace_disable_dedicated_storage_section(mainConfig); ApplyReplaceSettings(request, settings); - return RunSimple( + return RunSimple( std::move(request), - &Ydb::Config::V1::BSConfigService::Stub::AsyncReplaceConfig); + &Ydb::Config::V1::ConfigService::Stub::AsyncReplaceConfig); } TAsyncStatus ReplaceConfigEnableDedicatedStorageSection(const TString& mainConfig, const TString& storageConfig, const TReplaceConfigSettings& settings = {}) { @@ -55,9 +55,9 @@ class TConfigClient::TImpl : public TClientImplCommon { ApplyReplaceSettings(request, settings); - return RunSimple( + return RunSimple( std::move(request), - &Ydb::Config::V1::BSConfigService::Stub::AsyncReplaceConfig); + &Ydb::Config::V1::ConfigService::Stub::AsyncReplaceConfig); } TAsyncFetchConfigResult FetchConfig(const TFetchConfigSettings& settings = {}) { @@ -73,30 +73,30 @@ class TConfigClient::TImpl : public TClientImplCommon { switch (entry.identity().type_case()) { case Ydb::Config::ConfigIdentity::TypeCase::kMain: identity = TMainConfigIdentity { - .Version = entry.identity().version(); - .Cluster = entry.identity().cluster(); + .Version = entry.identity().version(), + .Cluster = entry.identity().cluster(), }; break; - case Ydb::Config::ConfigConfigIdentity::TypeCase::kStorage: + case Ydb::Config::ConfigIdentity::TypeCase::kStorage: identity = TStorageConfigIdentity { - .Version = entry.identity().version(); - .Cluster = entry.identity().cluster(); + .Version = entry.identity().version(), + .Cluster = entry.identity().cluster(), }; break; - case Ydb::Config::ConfigConfigIdentity::TypeCase::kDatabase: + case Ydb::Config::ConfigIdentity::TypeCase::kDatabase: identity = TDatabaseConfigIdentity { - .Version = entry.identity().version(); - .Cluster = entry.identity().cluster(); - .Database = entry.identity().database().database(); + .Version = entry.identity().version(), + .Cluster = entry.identity().cluster(), + .Database = entry.identity().database().database(), }; break; - case Ydb::Config::ConfigIdentity::TypeCase::ACTION_NOT_SET: + case Ydb::Config::ConfigIdentity::TypeCase::TYPE_NOT_SET: break; // leave in monostate; uknown identity } configs.push_back(TConfig{ - .Identity = std::monostate, - .Config = identity, + .Identity = identity, + .Config = entry.config(), }); } } @@ -105,10 +105,10 @@ class TConfigClient::TImpl : public TClientImplCommon { promise.SetValue(std::move(val)); }; - Connections_->RunDeferred( + Connections_->RunDeferred( std::move(request), extractor, - &Ydb::Config::V1::BSConfigService::Stub::AsyncFetchConfig, + &Ydb::Config::V1::ConfigService::Stub::AsyncFetchConfig, DbDriverState_, INITIAL_DEFERRED_CALL_DELAY, TRpcRequestSettings::Make(settings)); @@ -120,12 +120,12 @@ class TConfigClient::TImpl : public TClientImplCommon { auto request = MakeRequest(); request.set_self_assembly_uuid(selfAssemblyUUID); - return RunSimple(std::move(request), - &Ydb::Config::V1::BSConfigService::Stub::AsyncBootstrapCluster); + &Ydb::Config::V1::ConfigService::Stub::AsyncBootstrapCluster); } private: - static void ApplyReplaceSettings(auto& request, const TReplaceConfigSettings& settings) const { + static void ApplyReplaceSettings(auto& request, const TReplaceConfigSettings& settings) { request.set_dry_run(settings.DryRun_); request.set_allow_unknown_fields(settings.AllowUnknownFields_); request.set_bypass_checks(settings.BypassChecks_); @@ -165,7 +165,7 @@ TAsyncStatus TConfigClient::ReplaceConfigEnableDedicatedStorageSection( const TString& storageConfig, const TReplaceConfigSettings& settings) { - return Impl_->ReplaceConfigEnableDedicatedStorageSection(mainConfig, storageConfig); + return Impl_->ReplaceConfigEnableDedicatedStorageSection(mainConfig, storageConfig, settings); } TAsyncFetchConfigResult TConfigClient::FetchConfig(const TFetchConfigSettings& settings) { diff --git a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h index 35bc431eac5f..380f4a4ed899 100644 --- a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h +++ b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h @@ -41,7 +41,7 @@ struct TDatabaseConfigIdentity { TString Database; }; -using TKnownIdentityTypes = std::variant< +using TIdentityTypes = std::variant< std::monostate , TMainConfigIdentity , TStorageConfigIdentity @@ -73,7 +73,6 @@ using TAsyncFetchConfigResult = NThreading::TFuture; class TConfigClient { public: - explicit TConfigClient(const TDriver& driver, const TCommonClientSettings& settings = {}); ~TConfigClient(); From 0156589f81d5dd4f1b25345b353a7f77ff9a0832 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Mon, 17 Feb 2025 20:04:32 +0000 Subject: [PATCH 40/50] WIP --- ydb/core/grpc_services/rpc_config_base.h | 34 ++++++---- .../ydb_cli/commands/ydb_storage_config.cpp | 66 +++++++++++++------ .../sdk/cpp/client/ydb_config/ydb_config.cpp | 6 +- .../sdk/cpp/client/ydb_config/ydb_config.h | 12 +++- .../ydb-cpp-sdk/client/config/config.h | 12 +++- .../sdk/cpp/src/client/config/config.cpp | 6 +- 6 files changed, 91 insertions(+), 45 deletions(-) diff --git a/ydb/core/grpc_services/rpc_config_base.h b/ydb/core/grpc_services/rpc_config_base.h index e78c586c87ab..67dee85fa793 100644 --- a/ydb/core/grpc_services/rpc_config_base.h +++ b/ydb/core/grpc_services/rpc_config_base.h @@ -286,25 +286,31 @@ class TBSConfigRequestGrpc : public TRpcOperationRequestActorGet()->Record; if (record.HasClusterYaml()) { auto conf = ev->Get()->Record.GetClusterYaml(); - auto& config = *result.add_config(); - auto& identity = *config.mutable_identity(); auto metadata = NYamlConfig::GetMainMetadata(conf); - // TODO: !imp - identity.set_version(*metadata.Version); - identity.set_cluster(AppData()->ClusterName); - identity.mutable_main(); - config.set_config(conf); + if (metadata.Version && metadata.Cluster) { + auto& config = *result.add_config(); + auto& identity = *config.mutable_identity(); + identity.set_version(*metadata.Version); + identity.set_cluster(*metadata.Cluster); + identity.mutable_main(); + config.set_config(conf); + } else { + // impossible + } } if (record.HasStorageYaml()) { auto conf = ev->Get()->Record.GetStorageYaml(); - auto& config = *result.add_config(); - auto& identity = *config.mutable_identity(); auto metadata = NYamlConfig::GetStorageMetadata(conf); - // TODO: !imp - identity.set_version(*metadata.Version); - identity.set_cluster(AppData()->ClusterName); - identity.mutable_storage(); - config.set_config(conf); + if (metadata.Version && metadata.Cluster) { + auto& config = *result.add_config(); + auto& identity = *config.mutable_identity(); + identity.set_version(*metadata.Version); + identity.set_cluster(*metadata.Cluster); + identity.mutable_storage(); + config.set_config(conf); + } else { + // impossible + } } self->ReplyWithResult(Ydb::StatusIds::SUCCESS, result, self->ActorContext()); } else { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp b/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp index 6f93d6362755..c570ffd4f8df 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp @@ -63,24 +63,42 @@ void TCommandStorageConfigFetch::Parse(TConfig& config) { int TCommandStorageConfigFetch::Run(TConfig& config) { auto driver = std::make_unique(CreateDriver(config)); auto client = NYdb::NConfig::TConfigClient(*driver); - // FIXME - // auto result = client.FetchConfig(DedicatedStorageSection, DedicatedClusterSection).GetValueSync(); - auto result = client.FetchConfig().GetValueSync(); + + bool needDetach = DedicatedStorageSection || DedicatedClusterSection; + + NYdb::NConfig::TFetchAllConfigsSettings settings; + + if (needDetach) { + settings.Transform(NYdb::NConfig::EFetchAllConfigsTransform::DETACH_STORAGE_CONFIG_SECTION); + } + + auto result = client.FetchAllConfigs(settings).GetValueSync(); NStatusHelpers::ThrowOnError(result); - TString clusterConfig; // = result.GetMainConfig(); - TString storageConfig; // = result.GetStorageConfig(); + TString clusterConfig; + TString storageConfig; + + for (const auto& entry : result.GetConfigs()) { + std::visit([&](auto&& arg) { + using T = std::decay_t; + if constexpr (std::is_same_v) { + clusterConfig = entry.Config; + } else if constexpr (std::is_same_v) { + storageConfig = entry.Config; + } + }, entry.Identity); + } if (!clusterConfig.empty()) { if (!storageConfig.empty() || DedicatedStorageSection) { - Cout << "cluster config: " << Endl; + Cerr << "cluster config: " << Endl; } Cout << WrapYaml(TString(clusterConfig)); } if (!storageConfig.empty()) { if (!clusterConfig.empty() || DedicatedClusterSection) { - Cout << "storage config:" << Endl; + Cerr << "storage config:" << Endl; } Cout << WrapYaml(TString(storageConfig)); } @@ -142,20 +160,26 @@ void TCommandStorageConfigReplace::Parse(TConfig& config) { } int TCommandStorageConfigReplace::Run(TConfig& config) { - Y_UNUSED(config); // FIXME - // std::unique_ptr driver = std::make_unique(CreateDriver(config)); - // auto client = NYdb::NConfig::TConfigClient(*driver); - // NYdb::NConfig::TReplaceConfigSettings settings; - // settings - // .SwitchDedicatedStorageSection(SwitchDedicatedStorageSection) - // .DedicatedConfigMode(DedicatedConfigMode); - - // auto status = client.ReplaceConfig(ClusterYaml, StorageYaml, settings).GetValueSync(); - // NStatusHelpers::ThrowOnError(status); - - // if (!status.GetIssues()) { - // Cout << status << Endl; - // } + std::unique_ptr driver = std::make_unique(CreateDriver(config)); + auto client = NYdb::NConfig::TConfigClient(*driver); + + auto status = [&]() { + if (SwitchDedicatedStorageSection && *SwitchDedicatedStorageSection) { + return client.ReplaceConfigDisableDedicatedStorageSection(ClusterYaml.value()).GetValueSync(); + } else if (SwitchDedicatedStorageSection && !*SwitchDedicatedStorageSection) { + return client.ReplaceConfigEnableDedicatedStorageSection(ClusterYaml.value(), StorageYaml.value()).GetValueSync(); + } else if (DedicatedConfigMode) { + return client.ReplaceConfig(ClusterYaml.value(), StorageYaml.value()).GetValueSync(); + } else { + return client.ReplaceConfig(ClusterYaml.value()).GetValueSync(); + } + }(); + + NStatusHelpers::ThrowOnError(status); + + if (!status.GetIssues()) { + Cout << status << Endl; + } return EXIT_SUCCESS; } diff --git a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp index fc41c73cf3d9..dc97681a453c 100644 --- a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp +++ b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp @@ -60,7 +60,7 @@ class TConfigClient::TImpl : public TClientImplCommon { &Ydb::Config::V1::ConfigService::Stub::AsyncReplaceConfig); } - TAsyncFetchConfigResult FetchConfig(const TFetchConfigSettings& settings = {}) { + TAsyncFetchConfigResult FetchAllConfigs(const TFetchAllConfigsSettings& settings = {}) { auto request = MakeOperationRequest(settings); auto promise = NThreading::NewPromise(); @@ -168,8 +168,8 @@ TAsyncStatus TConfigClient::ReplaceConfigEnableDedicatedStorageSection( return Impl_->ReplaceConfigEnableDedicatedStorageSection(mainConfig, storageConfig, settings); } -TAsyncFetchConfigResult TConfigClient::FetchConfig(const TFetchConfigSettings& settings) { - return Impl_->FetchConfig(settings); +TAsyncFetchConfigResult TConfigClient::FetchAllConfigs(const TFetchAllConfigsSettings& settings) { + return Impl_->FetchAllConfigs(settings); } TAsyncStatus TConfigClient::BootstrapCluster( diff --git a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h index 380f4a4ed899..0ade45945aa8 100644 --- a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h +++ b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h @@ -21,7 +21,15 @@ struct TReplaceConfigSettings : public NYdb::TOperationRequestSettings {}; +enum class EFetchAllConfigsTransform { + NONE, + DETACH_STORAGE_CONFIG_SECTION, + ATTACH_STORAGE_CONFIG_SECTION, +}; + +struct TFetchAllConfigsSettings : public NYdb::TOperationRequestSettings { + FLUENT_SETTING(EFetchAllConfigsTransform, Transform); +}; struct TBootstrapClusterSettings : public NYdb::TOperationRequestSettings {}; @@ -100,7 +108,7 @@ class TConfigClient { const TReplaceConfigSettings& settings = {}); // Fetch current cluster storage config - TAsyncFetchConfigResult FetchConfig(const TFetchConfigSettings& settings = {}); + TAsyncFetchConfigResult FetchAllConfigs(const TFetchAllConfigsSettings& settings = {}); // Bootstrap cluster with automatic configuration TAsyncStatus BootstrapCluster( diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/config/config.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/config/config.h index da7bfd51b5c6..e4fdb38e5832 100644 --- a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/config/config.h +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/config/config.h @@ -17,7 +17,15 @@ struct TReplaceConfigSettings : public NYdb::TOperationRequestSettings {}; +enum class EFetchAllConfigsTransform { + NONE, + DETACH_STORAGE_CONFIG_SECTION, + ATTACH_STORAGE_CONFIG_SECTION, +}; + +struct TFetchAllConfigsSettings : public NYdb::TOperationRequestSettings { + FLUENT_SETTING(EFetchAllConfigsTransform, Transform); +}; struct TBootstrapClusterSettings : public NYdb::TOperationRequestSettings {}; @@ -96,7 +104,7 @@ class TConfigClient { const TReplaceConfigSettings& settings = {}); // Fetch current cluster storage config - TAsyncFetchConfigResult FetchConfig(const TFetchConfigSettings& settings = {}); + TAsyncFetchConfigResult FetchAllConfigs(const TFetchAllConfigsSettings& settings = {}); // Bootstrap cluster with automatic configuration TAsyncStatus BootstrapCluster( diff --git a/ydb/public/sdk/cpp/src/client/config/config.cpp b/ydb/public/sdk/cpp/src/client/config/config.cpp index 187ef46d1cc9..8c645c737778 100644 --- a/ydb/public/sdk/cpp/src/client/config/config.cpp +++ b/ydb/public/sdk/cpp/src/client/config/config.cpp @@ -62,7 +62,7 @@ class TConfigClient::TImpl : public TClientImplCommon { &Ydb::Config::V1::ConfigService::Stub::AsyncReplaceConfig); } - TAsyncFetchConfigResult FetchConfig(const TFetchConfigSettings& settings = {}) { + TAsyncFetchConfigResult FetchAllConfigs(const TFetchAllConfigsSettings& settings = {}) { auto request = MakeOperationRequest(settings); auto promise = NThreading::NewPromise(); @@ -170,8 +170,8 @@ TAsyncStatus TConfigClient::ReplaceConfigEnableDedicatedStorageSection( return Impl_->ReplaceConfigEnableDedicatedStorageSection(mainConfig, storageConfig, settings); } -TAsyncFetchConfigResult TConfigClient::FetchConfig(const TFetchConfigSettings& settings) { - return Impl_->FetchConfig(settings); +TAsyncFetchConfigResult TConfigClient::FetchAllConfigs(const TFetchAllConfigsSettings& settings) { + return Impl_->FetchAllConfigs(settings); } TAsyncStatus TConfigClient::BootstrapCluster( From 8a86cc2c44fcf376fadf4b6e765a09d1ca77ffe1 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Tue, 18 Feb 2025 18:07:16 +0000 Subject: [PATCH 41/50] WIP --- .../blobstorage/nodewarden/distconf_validate.cpp | 2 +- ydb/core/cms/ui/yaml_config.js | 14 +++++++------- ydb/library/yaml_config/yaml_config_parser.cpp | 1 + .../lib/ydb_cli/commands/ydb_dynamic_config.cpp | 5 +---- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/ydb/core/blobstorage/nodewarden/distconf_validate.cpp b/ydb/core/blobstorage/nodewarden/distconf_validate.cpp index 662b4e0a99b2..e056f1d7ec81 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_validate.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_validate.cpp @@ -179,7 +179,7 @@ namespace NKikimr::NStorage { } else if (!pdisk.HasPDiskID()) { return "PDiskID field missing"; } else if (!pdisk.HasPath()) { - return "Path field missing"; + return config.DebugString() + "Path field missing"; } else if (!pdisk.HasPDiskGuid()) { return "PDiskGuid field missing"; } else if (!pdisk.HasPDiskCategory()) { diff --git a/ydb/core/cms/ui/yaml_config.js b/ydb/core/cms/ui/yaml_config.js index 37f1625211cb..69a05ce616f0 100644 --- a/ydb/core/cms/ui/yaml_config.js +++ b/ydb/core/cms/ui/yaml_config.js @@ -235,16 +235,16 @@ class YamlConfigState { onYamlLoaded(success, data) { if (success) { - this.cluster = data.Response.identity.cluster; - $('#yaml-cluster').text(data.Response.identity.cluster); + this.cluster = data.Response.identity[0].cluster; + $('#yaml-cluster').text(data.Response.identity[0].cluster); - this.version = data.Response.identity.version; - $('#yaml-version').text(data.Response.identity.version); + this.version = data.Response.identity[0].version; + $('#yaml-version').text(data.Response.identity[0].version); - if (this.config !== data.Response.config) { - this.codeMirror.setValue((data.Response.config !== undefined) ? data.Response.config[0] : ""); + if (this.config !== data.Response.config[0]) { + this.codeMirror.setValue((data.Response.config[0] !== undefined) ? data.Response.config[0] : ""); this.codeMirror.trigger('fold', 'editor.foldLevel2'); - this.config = data.Response.config; + this.config = data.Response.config[0]; } if (data.Response.volatile_configs === undefined) { diff --git a/ydb/library/yaml_config/yaml_config_parser.cpp b/ydb/library/yaml_config/yaml_config_parser.cpp index 17611786046b..6cf63bab21c1 100644 --- a/ydb/library/yaml_config/yaml_config_parser.cpp +++ b/ydb/library/yaml_config/yaml_config_parser.cpp @@ -544,6 +544,7 @@ namespace NKikimr::NYaml { ui64 nextHostConfigID = 1; + // TODO: validate all host_configs exists (or better just drop this legacy and use yaml anchors) // Find the next available host_config_id if (ephemeralConfig.HostConfigsSize()) { for(const auto& hostConfig : ephemeralConfig.GetHostConfigs()) { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp index cdd0abc1b188..6e8057d7a1df 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp @@ -215,10 +215,7 @@ int TCommandConfigReplace::Run(TConfig& config) { // TODO absent database - auto status = client.ReplaceConfig( - DynamicConfig, - {}, - settings).GetValueSync(); + auto status = client.ReplaceConfig(DynamicConfig, settings).GetValueSync(); if (status.IsUnimplementedError()) { Cerr << "Warning: Fallback to DynamicConfig API" << Endl; From bc23f9f151cfee8cd7500f9cdea65c7ec6604b23 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Tue, 18 Feb 2025 18:12:23 +0000 Subject: [PATCH 42/50] WIP --- ydb/core/blobstorage/nodewarden/distconf_validate.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/core/blobstorage/nodewarden/distconf_validate.cpp b/ydb/core/blobstorage/nodewarden/distconf_validate.cpp index e056f1d7ec81..662b4e0a99b2 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_validate.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_validate.cpp @@ -179,7 +179,7 @@ namespace NKikimr::NStorage { } else if (!pdisk.HasPDiskID()) { return "PDiskID field missing"; } else if (!pdisk.HasPath()) { - return config.DebugString() + "Path field missing"; + return "Path field missing"; } else if (!pdisk.HasPDiskGuid()) { return "PDiskGuid field missing"; } else if (!pdisk.HasPDiskCategory()) { From df75b3ad777508bf48646dd1b1f2053f325dd4c5 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Tue, 18 Feb 2025 18:46:24 +0000 Subject: [PATCH 43/50] WIP --- ydb/core/grpc_services/rpc_config.cpp | 135 ++++++++------------------ 1 file changed, 41 insertions(+), 94 deletions(-) diff --git a/ydb/core/grpc_services/rpc_config.cpp b/ydb/core/grpc_services/rpc_config.cpp index 703da58eeedf..0b4d7d402b98 100644 --- a/ydb/core/grpc_services/rpc_config.cpp +++ b/ydb/core/grpc_services/rpc_config.cpp @@ -25,31 +25,54 @@ using TEvBootstrapClusterRequest = using namespace NActors; using namespace Ydb; -bool CopyToConfigRequest(const Ydb::Config::ReplaceConfigRequest &from, NKikimrBlobStorage::TConfigRequest *to) { - TString configStr; +struct BSConfigApiShim { + std::optional SwitchDedicatedStorageSection; + std::optional MainConfig; + std::optional StorageConfig; + bool DedicatedConfigMode = false; +}; + +BSConfigApiShim ConvertConfigReplaceRequest(const auto& request) { + BSConfigApiShim result; auto fillConfigs = [&](const auto& configBundle) { - configStr = configBundle.main_config(); + if (configBundle.has_main_config()) { + result.MainConfig = configBundle.main_config(); + } + + if (configBundle.has_storage_config()) { + result.StorageConfig = configBundle.storage_config(); + } }; - switch (from.action_case()) { + switch (request.action_case()) { case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceEnableDedicatedStorageSection: - fillConfigs(from.replace_enable_dedicated_storage_section()); + result.SwitchDedicatedStorageSection = true; + result.DedicatedConfigMode = true; + fillConfigs(request.replace_enable_dedicated_storage_section()); break; case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceDisableDedicatedStorageSection: - configStr = from.replace_disable_dedicated_storage_section(); + result.SwitchDedicatedStorageSection = false; + result.MainConfig = request.replace_disable_dedicated_storage_section(); break; case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceWithDedicatedStorageSection: - fillConfigs(from.replace_with_dedicated_storage_section()); + result.DedicatedConfigMode = true; + fillConfigs(request.replace_with_dedicated_storage_section()); break; case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplace: - configStr = from.replace(); + result.MainConfig = request.replace(); break; case Ydb::Config::ReplaceConfigRequest::ActionCase::ACTION_NOT_SET: break; // TODO: handle as error? } - to->CopyFrom(NKikimr::NYaml::BuildInitDistributedStorageCommand(configStr)); + return result; +} + +bool CopyToConfigRequest(const Ydb::Config::ReplaceConfigRequest &from, NKikimrBlobStorage::TConfigRequest *to) { + auto shim = ConvertConfigReplaceRequest(from); + + to->CopyFrom(NKikimr::NYaml::BuildInitDistributedStorageCommand(shim.MainConfig.value_or(TString{}))); return true; } @@ -127,30 +150,9 @@ class TReplaceStorageConfigRequest : public TBSConfigRequestGrpcaction_case()) { - case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceEnableDedicatedStorageSection: - fillConfigs(GetProtoRequest()->replace_enable_dedicated_storage_section()); - break; - case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceDisableDedicatedStorageSection: - configStr = GetProtoRequest()->replace_disable_dedicated_storage_section(); - break; - case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceWithDedicatedStorageSection: - fillConfigs(GetProtoRequest()->replace_with_dedicated_storage_section()); - break; - case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplace: - configStr = GetProtoRequest()->replace(); - break; - case Ydb::Config::ReplaceConfigRequest::ActionCase::ACTION_NOT_SET: - break; // TODO: handle as error? - } + auto shim = ConvertConfigReplaceRequest(*GetProtoRequest()); - cmd->SetYAML(configStr); + cmd->SetYAML(shim.MainConfig.value_or(TString{})); } void FillDistconfResult(NKikimrBlobStorage::TEvNodeConfigInvokeOnRootResult& /*record*/, @@ -160,29 +162,8 @@ class TReplaceStorageConfigRequest : public TBSConfigRequestGrpcaction_case()) { - case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceEnableDedicatedStorageSection: - fillConfigs(GetProtoRequest()->replace_enable_dedicated_storage_section()); - break; - case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceDisableDedicatedStorageSection: - configStr = GetProtoRequest()->replace_disable_dedicated_storage_section(); - break; - case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceWithDedicatedStorageSection: - fillConfigs(GetProtoRequest()->replace_with_dedicated_storage_section()); - break; - case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplace: - configStr = GetProtoRequest()->replace(); - break; - case Ydb::Config::ReplaceConfigRequest::ActionCase::ACTION_NOT_SET: - break; // TODO: handle as error? - } - newConfig = NYaml::Parse(configStr); // check allow unknown fields + auto shim = ConvertConfigReplaceRequest(*GetProtoRequest()); + newConfig = NYaml::Parse(shim.MainConfig.value_or(TString{})); // TODO: !imp check allow unknown fields } catch (const std::exception&) { return false; // assuming no distconf enabled in this config } @@ -192,47 +173,13 @@ class TReplaceStorageConfigRequest : public TBSConfigRequestGrpc ProcessControllerQuery() override { auto *request = GetProtoRequest(); - std::optional switch_dedicated_storage_section; - std::optional mainConfig; - std::optional storageConfig; - bool dedicated_config_mode = false; - - auto fillConfigs = [&](const auto& configBundle) { - if (configBundle.has_main_config()) { - mainConfig = configBundle.main_config(); - } - - if (configBundle.has_storage_config()) { - storageConfig = configBundle.storage_config(); - } - }; - - switch (request->action_case()) { - case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceEnableDedicatedStorageSection: - switch_dedicated_storage_section = true; - dedicated_config_mode = true; - fillConfigs(request->replace_enable_dedicated_storage_section()); - break; - case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceDisableDedicatedStorageSection: - switch_dedicated_storage_section = false; - mainConfig = request->replace_disable_dedicated_storage_section(); - break; - case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceWithDedicatedStorageSection: - dedicated_config_mode = true; - fillConfigs(request->replace_with_dedicated_storage_section()); - break; - case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplace: - mainConfig = request->replace(); - break; - case Ydb::Config::ReplaceConfigRequest::ActionCase::ACTION_NOT_SET: - break; // TODO: handle as error? - } + auto shim = ConvertConfigReplaceRequest(*request); return std::make_unique( - mainConfig, - storageConfig, - switch_dedicated_storage_section, - dedicated_config_mode, + shim.MainConfig, + shim.StorageConfig, + shim.SwitchDedicatedStorageSection, + shim.DedicatedConfigMode, request->allow_unknown_fields() || request->bypass_checks(), request->bypass_checks(), request->bypass_checks()); From e2e8a8edc84e628c0d2849bc206abfd26135d066 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Tue, 18 Feb 2025 19:02:34 +0000 Subject: [PATCH 44/50] WIP --- ydb/core/grpc_services/rpc_config.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ydb/core/grpc_services/rpc_config.cpp b/ydb/core/grpc_services/rpc_config.cpp index 0b4d7d402b98..dcfd6b299324 100644 --- a/ydb/core/grpc_services/rpc_config.cpp +++ b/ydb/core/grpc_services/rpc_config.cpp @@ -152,7 +152,16 @@ class TReplaceStorageConfigRequest : public TBSConfigRequestGrpcSetYAML(shim.MainConfig.value_or(TString{})); + if (shim.MainConfig) { + cmd->SetYAML(*shim.MainConfig); + } + if (shim.StorageConfig) { + cmd->SetStorageYAML(*shim.StorageConfig); + } + if (shim.SwitchDedicatedStorageSection) { + cmd->SetSwitchDedicatedStorageSection(*shim.SwitchDedicatedStorageSection); + } + cmd->SetDedicatedStorageSectionConfigMode(shim.DedicatedConfigMode); } void FillDistconfResult(NKikimrBlobStorage::TEvNodeConfigInvokeOnRootResult& /*record*/, From 1a3da7571916dcd0cd2b2c903dedee81b6db44dc Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Tue, 18 Feb 2025 19:15:30 +0000 Subject: [PATCH 45/50] WIP --- ydb/core/grpc_services/rpc_config.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ydb/core/grpc_services/rpc_config.cpp b/ydb/core/grpc_services/rpc_config.cpp index dcfd6b299324..2553ede43682 100644 --- a/ydb/core/grpc_services/rpc_config.cpp +++ b/ydb/core/grpc_services/rpc_config.cpp @@ -128,9 +128,7 @@ void CopyFromConfigResponse(const NKikimrBlobStorage::TConfigResponse &from, Ydb auto& identity = *config.mutable_identity(); identity.set_version(itemConfigGeneration); identity.set_cluster(AppData()->ClusterName); - identity.mutable_storage(); - // TODO: !imp fill metadata ? - // are we sure that it is storage config? + identity.mutable_main(); config.set_config(NYaml::ParseProtoToYaml(storageConfig)); } @@ -143,6 +141,7 @@ class TReplaceStorageConfigRequest : public TBSConfigRequestGrpcClusterName); - identity.mutable_main(); // TODO: is it really main ? + identity.mutable_main(); config.set_config(conf); } From 8c98eacabd35b71945d5583cbbf34f9f43770a99 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Tue, 18 Feb 2025 19:16:56 +0000 Subject: [PATCH 46/50] WIP --- ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp index 6e8057d7a1df..b3087bb1ad37 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp @@ -213,8 +213,6 @@ int TCommandConfigReplace::Run(TConfig& config) { settings.AllowUnknownFields(); } - // TODO absent database - auto status = client.ReplaceConfig(DynamicConfig, settings).GetValueSync(); if (status.IsUnimplementedError()) { From a4b73f71f22568bc5bef8ab3745ca3d5b6ef2d92 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Tue, 18 Feb 2025 20:19:30 +0000 Subject: [PATCH 47/50] WIP --- ydb/core/cms/console/console_handshake.cpp | 15 ++++++++++++++- ydb/core/grpc_services/rpc_config.cpp | 19 ++++++++++++++++--- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/ydb/core/cms/console/console_handshake.cpp b/ydb/core/cms/console/console_handshake.cpp index a8bdde7694bc..aa269b63b1a2 100644 --- a/ydb/core/cms/console/console_handshake.cpp +++ b/ydb/core/cms/console/console_handshake.cpp @@ -183,10 +183,23 @@ void TConfigsManager::Handle(TEvBlobStorage::TEvControllerValidateConfigRequest: } auto& record = response->Record; + + bool allowIncorrectVersion = ev->Get()->Record.GetAllowIncorrectVersion(); + bool allowIncorrectCluster = ev->Get()->Record.GetAllowIncorrectCluster(); + + if (allowIncorrectVersion xor allowIncorrectCluster) { + record.SetStatus(NKikimrBlobStorage::TEvControllerValidateConfigResponse::ConfigNotValid); + record.SetErrorReason("Options AllowIncorrectVersion and AllowIncorrectCluster currently can be used only together"); + SendInReply(ev->Sender, ev->InterconnectSession, std::move(response), ev->Cookie); + return; + } + + bool force = allowIncorrectVersion && allowIncorrectCluster; + auto mainYamlConfig = ev->Get()->Record.GetYAML(); TUpdateConfigOpContext opCtx; - ReplaceMainConfigMetadata(mainYamlConfig, false, opCtx); + ReplaceMainConfigMetadata(mainYamlConfig, force, opCtx); ValidateMainConfig(opCtx); bool hasForbiddenUnknownFields = !opCtx.UnknownFields.empty() && !ev->Get()->Record.GetAllowUnknownFields(); diff --git a/ydb/core/grpc_services/rpc_config.cpp b/ydb/core/grpc_services/rpc_config.cpp index 2553ede43682..418248e92723 100644 --- a/ydb/core/grpc_services/rpc_config.cpp +++ b/ydb/core/grpc_services/rpc_config.cpp @@ -138,7 +138,13 @@ class TReplaceStorageConfigRequest : public TBSConfigRequestGrpc; using TBase::TBase; - bool ValidateRequest(Ydb::StatusIds::StatusCode& /*status*/, NYql::TIssues& /*issues*/) override { + bool ValidateRequest(Ydb::StatusIds::StatusCode& status, NYql::TIssues& issues) override { + const auto& request = *GetProtoRequest(); + if (request.dry_run()) { + status = Ydb::StatusIds::BAD_REQUEST; + issues.AddIssue("DryRun is not supported yet."); + return false; + } return true; } @@ -200,9 +206,16 @@ class TFetchStorageConfigRequest : public TBSConfigRequestGrpc; using TBase::TBase; - bool ValidateRequest(Ydb::StatusIds::StatusCode& /*status*/, NYql::TIssues& /*issues*/) override { + bool ValidateRequest(Ydb::StatusIds::StatusCode& status, NYql::TIssues& issues) override { + const auto& request = *GetProtoRequest(); + if (request.mode_case() != Ydb::Config::FetchConfigRequest::ModeCase::kAll) { + status = Ydb::StatusIds::BAD_REQUEST; + issues.AddIssue("Only fetch mode \"all\" is supported now."); + return false; + } return true; } + NACLib::EAccessRights GetRequiredAccessRights() const { return NACLib::GenericManage; } @@ -241,7 +254,7 @@ class TFetchStorageConfigRequest : public TBSConfigRequestGrpc Date: Tue, 18 Feb 2025 20:20:53 +0000 Subject: [PATCH 48/50] WIP --- ydb/core/mind/bscontroller/console_interaction.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/core/mind/bscontroller/console_interaction.cpp b/ydb/core/mind/bscontroller/console_interaction.cpp index 09d011584923..01f51524d8fc 100644 --- a/ydb/core/mind/bscontroller/console_interaction.cpp +++ b/ydb/core/mind/bscontroller/console_interaction.cpp @@ -187,7 +187,7 @@ namespace NKikimr::NBsController { case NKikimrBlobStorage::TEvControllerConsoleCommitResponse::NotCommitted: STLOG(PRI_CRIT, BS_CONTROLLER, BSC28, "Console config not committed"); - Y_DEBUG_ABORT_S(record.GetErrorReason()); + Y_DEBUG_ABORT_S(record.GetErrorReason()); // FIXME: fails here on force break; case NKikimrBlobStorage::TEvControllerConsoleCommitResponse::Committed: From eb368c97e9345acc52ff9941dc06936e0cb73cac Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Tue, 18 Feb 2025 21:24:35 +0000 Subject: [PATCH 49/50] WIP --- ydb/core/blobstorage/nodewarden/distconf_console.cpp | 2 +- ydb/core/grpc_services/rpc_config.cpp | 4 +++- ydb/library/yaml_config/yaml_config_parser.h | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ydb/core/blobstorage/nodewarden/distconf_console.cpp b/ydb/core/blobstorage/nodewarden/distconf_console.cpp index 71f9c514f6e5..ee125a8a53bb 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_console.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_console.cpp @@ -103,7 +103,7 @@ namespace NKikimr::NStorage { } NTabletPipe::SendData(SelfId(), ConsolePipeId, new TEvBlobStorage::TEvControllerConsoleCommitRequest( - MainConfigYaml), // FIXME + MainConfigYaml), // FIXME: probably should propagate force here ++CommitRequestCookie); break; } diff --git a/ydb/core/grpc_services/rpc_config.cpp b/ydb/core/grpc_services/rpc_config.cpp index 418248e92723..9eaf6c8f221d 100644 --- a/ydb/core/grpc_services/rpc_config.cpp +++ b/ydb/core/grpc_services/rpc_config.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include "rpc_config_base.h" #include @@ -177,7 +178,8 @@ class TReplaceStorageConfigRequest : public TBSConfigRequestGrpc Date: Wed, 19 Feb 2025 10:28:58 +0000 Subject: [PATCH 50/50] simplify --- .../base/blobstorage_console_events.h | 12 ++---- ydb/core/cms/console/console_handshake.cpp | 39 ++++--------------- ydb/core/grpc_services/rpc_config.cpp | 1 - .../mind/bscontroller/console_interaction.cpp | 12 ++---- .../mind/bscontroller/console_interaction.h | 3 +- ydb/core/protos/blobstorage.proto | 9 ++--- 6 files changed, 20 insertions(+), 56 deletions(-) diff --git a/ydb/core/blobstorage/base/blobstorage_console_events.h b/ydb/core/blobstorage/base/blobstorage_console_events.h index f6e27f97a078..47d3932fa2af 100644 --- a/ydb/core/blobstorage/base/blobstorage_console_events.h +++ b/ydb/core/blobstorage/base/blobstorage_console_events.h @@ -35,13 +35,11 @@ namespace NKikimr { TEvControllerConsoleCommitRequest( const TString& yamlConfig, bool allowUnknownFields = false, - bool allowIncorrectVersion = false, - bool allowIncorrectCluster = false) { + bool bypassMetadataChecks = false) { Record.SetYAML(yamlConfig); Record.SetAllowUnknownFields(allowUnknownFields); - Record.SetAllowIncorrectVersion(allowIncorrectVersion); - Record.SetAllowIncorrectCluster(allowIncorrectCluster); + Record.SetBypassMetadataChecks(bypassMetadataChecks); } TString ToString() const override { @@ -90,8 +88,7 @@ namespace NKikimr { std::optional switchDedicatedStorageSection, bool dedicatedConfigMode, bool allowUnknownFields, - bool allowIncorrectVersion, - bool allowIncorrectCluster) { + bool bypassMetadataChecks) { if (clusterYaml) { Record.SetClusterYaml(*clusterYaml); @@ -104,8 +101,7 @@ namespace NKikimr { } Record.SetDedicatedConfigMode(dedicatedConfigMode); Record.SetAllowUnknownFields(allowUnknownFields); - Record.SetAllowIncorrectVersion(allowIncorrectVersion); - Record.SetAllowIncorrectCluster(allowIncorrectCluster); + Record.SetBypassMetadataChecks(bypassMetadataChecks); } TString ToString() const override { diff --git a/ydb/core/cms/console/console_handshake.cpp b/ydb/core/cms/console/console_handshake.cpp index aa269b63b1a2..ad89f4d8008a 100644 --- a/ydb/core/cms/console/console_handshake.cpp +++ b/ydb/core/cms/console/console_handshake.cpp @@ -20,35 +20,25 @@ class TConfigsManager::TConsoleCommitActor : public TActorBootstrapped(); - response->Record.SetStatus(NKikimrBlobStorage::TEvControllerConsoleCommitResponse::NotCommitted); - response->Record.SetErrorReason("Options AllowIncorrectVersion and AllowIncorrectCluster currently can be used only together"); - SendInReply(std::move(response)); - PassAway(); - } - auto executeRequest = [&](auto& request) { request->Record.MutableRequest()->set_config(MainYamlConfig); request->Record.MutableRequest()->set_allow_unknown_fields(AllowUnknownFields); Send(consoleId, request.release()); }; - if (AllowIncorrectVersion && AllowIncorrectCluster) { + if (BypassMetadataChecks) { auto request = std::make_unique(); executeRequest(request); } else { @@ -92,8 +82,7 @@ class TConfigsManager::TConsoleCommitActor : public TActorBootstrappedGet()->Record; const auto& mainYamlConfig = record.GetYAML(); bool allowUnknownFields = record.GetAllowUnknownFields(); - bool allowIncorrectVersion = record.GetAllowIncorrectVersion(); - bool allowIncorrectCluster = record.GetAllowIncorrectCluster(); + bool bypassMetadataChecks = record.GetBypassMetadataChecks(); if (!CheckSession(*ev, response, NKikimrBlobStorage::TEvControllerConsoleCommitResponse::SessionMismatch)) { return; } @@ -169,8 +157,7 @@ void TConfigsManager::Handle(TEvBlobStorage::TEvControllerConsoleCommitRequest:: ev->Sender, mainYamlConfig, allowUnknownFields, - allowIncorrectVersion, - allowIncorrectCluster, + bypassMetadataChecks, ev->InterconnectSession, ev->Cookie); CommitActor = Register(actor); @@ -184,22 +171,12 @@ void TConfigsManager::Handle(TEvBlobStorage::TEvControllerValidateConfigRequest: auto& record = response->Record; - bool allowIncorrectVersion = ev->Get()->Record.GetAllowIncorrectVersion(); - bool allowIncorrectCluster = ev->Get()->Record.GetAllowIncorrectCluster(); - - if (allowIncorrectVersion xor allowIncorrectCluster) { - record.SetStatus(NKikimrBlobStorage::TEvControllerValidateConfigResponse::ConfigNotValid); - record.SetErrorReason("Options AllowIncorrectVersion and AllowIncorrectCluster currently can be used only together"); - SendInReply(ev->Sender, ev->InterconnectSession, std::move(response), ev->Cookie); - return; - } - - bool force = allowIncorrectVersion && allowIncorrectCluster; + bool bypassMetadataChecks = ev->Get()->Record.GetBypassMetadataChecks(); auto mainYamlConfig = ev->Get()->Record.GetYAML(); TUpdateConfigOpContext opCtx; - ReplaceMainConfigMetadata(mainYamlConfig, force, opCtx); + ReplaceMainConfigMetadata(mainYamlConfig, bypassMetadataChecks, opCtx); ValidateMainConfig(opCtx); bool hasForbiddenUnknownFields = !opCtx.UnknownFields.empty() && !ev->Get()->Record.GetAllowUnknownFields(); diff --git a/ydb/core/grpc_services/rpc_config.cpp b/ydb/core/grpc_services/rpc_config.cpp index 9eaf6c8f221d..2f4f8ce81cea 100644 --- a/ydb/core/grpc_services/rpc_config.cpp +++ b/ydb/core/grpc_services/rpc_config.cpp @@ -197,7 +197,6 @@ class TReplaceStorageConfigRequest : public TBSConfigRequestGrpcallow_unknown_fields() || request->bypass_checks(), - request->bypass_checks(), request->bypass_checks()); } }; diff --git a/ydb/core/mind/bscontroller/console_interaction.cpp b/ydb/core/mind/bscontroller/console_interaction.cpp index 01f51524d8fc..1a8d739cf327 100644 --- a/ydb/core/mind/bscontroller/console_interaction.cpp +++ b/ydb/core/mind/bscontroller/console_interaction.cpp @@ -95,8 +95,7 @@ namespace NKikimr::NBsController { new TEvBlobStorage::TEvControllerConsoleCommitRequest( yaml, AllowUnknownFields, - AllowIncorrectVersion, - AllowIncorrectCluster)); + BypassMetadataChecks)); } } break; @@ -158,8 +157,7 @@ namespace NKikimr::NBsController { new TEvBlobStorage::TEvControllerConsoleCommitRequest( yaml, AllowUnknownFields, - AllowIncorrectVersion, - AllowIncorrectCluster)); + BypassMetadataChecks)); } } else { Y_ABORT_UNLESS(!ClientId); @@ -274,8 +272,7 @@ namespace NKikimr::NBsController { // don't need to reset them explicitly // every time we get new request we just replace them AllowUnknownFields = record.GetAllowUnknownFields(); - AllowIncorrectVersion = record.GetAllowIncorrectVersion(); - AllowIncorrectCluster = record.GetAllowIncorrectCluster(); + BypassMetadataChecks = record.GetBypassMetadataChecks(); } else { PendingYamlConfig.reset(); } @@ -308,8 +305,7 @@ namespace NKikimr::NBsController { auto validateConfigEv = std::make_unique(); validateConfigEv->Record.SetYAML(record.GetClusterYaml()); validateConfigEv->Record.SetAllowUnknownFields(record.GetAllowUnknownFields()); - validateConfigEv->Record.SetAllowIncorrectVersion(record.GetAllowIncorrectVersion()); - validateConfigEv->Record.SetAllowIncorrectCluster(record.GetAllowIncorrectCluster()); + validateConfigEv->Record.SetBypassMetadataChecks(record.GetBypassMetadataChecks()); NTabletPipe::SendData(Self.SelfId(), ConsolePipe, validateConfigEv.release()); } diff --git a/ydb/core/mind/bscontroller/console_interaction.h b/ydb/core/mind/bscontroller/console_interaction.h index ae910692b450..5002d5739fe4 100644 --- a/ydb/core/mind/bscontroller/console_interaction.h +++ b/ydb/core/mind/bscontroller/console_interaction.h @@ -48,8 +48,7 @@ namespace NKikimr::NBsController { std::optional PendingYamlConfig; bool AllowUnknownFields = false; - bool AllowIncorrectVersion = false; - bool AllowIncorrectCluster = false; + bool BypassMetadataChecks = false; std::optional> PendingStorageYamlConfig; diff --git a/ydb/core/protos/blobstorage.proto b/ydb/core/protos/blobstorage.proto index 0314599df6f1..87552ea39b1e 100644 --- a/ydb/core/protos/blobstorage.proto +++ b/ydb/core/protos/blobstorage.proto @@ -1426,8 +1426,7 @@ message TEvControllerProposeConfigResponse { message TEvControllerConsoleCommitRequest { optional string YAML = 1; optional bool AllowUnknownFields = 2; - optional bool AllowIncorrectVersion = 3; - optional bool AllowIncorrectCluster = 4; + optional bool BypassMetadataChecks = 3; } message TEvControllerConsoleCommitResponse { @@ -1450,8 +1449,7 @@ message TEvControllerReplaceConfigRequest { optional bool DedicatedConfigMode = 7; // console flags optional bool AllowUnknownFields = 8; - optional bool AllowIncorrectVersion = 9; - optional bool AllowIncorrectCluster = 10; + optional bool BypassMetadataChecks = 9; } message TEvControllerReplaceConfigResponse { @@ -1472,8 +1470,7 @@ message TEvControllerReplaceConfigResponse { message TEvControllerValidateConfigRequest { optional string YAML = 1; optional bool AllowUnknownFields = 2; - optional bool AllowIncorrectVersion = 3; - optional bool AllowIncorrectCluster = 4; + optional bool BypassMetadataChecks = 3; } message TEvControllerValidateConfigResponse {