From fdd6a06ac48cf4b5c810b5306244b6016c51d3d4 Mon Sep 17 00:00:00 2001 From: Ivan Sukhov Date: Fri, 30 May 2025 15:51:12 +0300 Subject: [PATCH 1/5] New _EnableRuntimeListing config parameter (#18968) Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../yql/providers/solomon/provider/yql_solomon_config.cpp | 1 + .../yql/providers/solomon/provider/yql_solomon_config.h | 1 + .../solomon/provider/yql_solomon_datasource_type_ann.cpp | 5 +++++ .../solomon/provider/yql_solomon_dq_integration.cpp | 1 + 4 files changed, 8 insertions(+) diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_config.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_config.cpp index 3cb7e646688c..9485f423bde2 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_config.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_config.cpp @@ -7,6 +7,7 @@ using namespace NCommon; TSolomonConfiguration::TSolomonConfiguration() { REGISTER_SETTING(*this, _EnableReading); + REGISTER_SETTING(*this, _EnableRuntimeListing); REGISTER_SETTING(*this, MetricsQueuePageSize); REGISTER_SETTING(*this, MetricsQueuePrefetchSize); REGISTER_SETTING(*this, MetricsQueueBatchCountLimit); diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h b/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h index fa6449839009..e8cbeb0d1b73 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h @@ -11,6 +11,7 @@ struct TSolomonSettings { using TConstPtr = std::shared_ptr; NCommon::TConfSetting _EnableReading; + NCommon::TConfSetting _EnableRuntimeListing; NCommon::TConfSetting MetricsQueuePageSize; NCommon::TConfSetting MetricsQueuePrefetchSize; NCommon::TConfSetting MetricsQueueBatchCountLimit; diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_type_ann.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_type_ann.cpp index 62fdbc211abd..6cf6e144e7ab 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_type_ann.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_type_ann.cpp @@ -91,6 +91,11 @@ class TSolomonDataSourceTypeAnnotationTransformer : public TVisitorTransformerBa } bool hasSelectors = !selectors.Content().empty(); + if (hasSelectors && !State_->Configuration->_EnableRuntimeListing.Get().GetOrElse(false)) { + ctx.AddError(TIssue(ctx.GetPosition(selectors.Pos()), "runtime listing is disabled, use `program` parameter")); + return TStatus::Error; + } + auto& program = *input->Child(TSoSourceSettings::idx_Program); if (!EnsureAtom(program, ctx)) { return TStatus::Error; diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp index 62b201d2b216..14976279eab4 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp @@ -367,6 +367,7 @@ class TSolomonDqIntegration: public TDqIntegrationBase { NDq::TDqSolomonReadParams readParams{ .Source = source }; + YQL_ENSURE(NActors::TlsActivationContext); auto metricsQueueActor = NActors::TActivationContext::ActorSystem()->Register( NDq::CreateSolomonMetricsQueueActor( std::min(maxTasksPerStage, totalMetricsCount), From 87ed0916314a4e80f5eccba0b77b03b3b9dba1e4 Mon Sep 17 00:00:00 2001 From: Ivan Sukhov Date: Fri, 18 Jul 2025 12:26:36 +0300 Subject: [PATCH 2/5] api inflight config for solomon data source & a minor bugfix (#21256) --- .../solomon/actors/dq_solomon_read_actor.cpp | 2 +- .../solomon/provider/yql_solomon_config.cpp | 1 + .../solomon/provider/yql_solomon_config.h | 1 + .../provider/yql_solomon_dq_integration.cpp | 9 ++++++--- .../client/solomon_accessor_client.cpp | 20 ++++++++++++++----- .../tools/solomon_emulator_grpc/__main__.py | 1 - 6 files changed, 24 insertions(+), 10 deletions(-) diff --git a/ydb/library/yql/providers/solomon/actors/dq_solomon_read_actor.cpp b/ydb/library/yql/providers/solomon/actors/dq_solomon_read_actor.cpp index 4bb7f7694325..3d4e6df214c9 100644 --- a/ydb/library/yql/providers/solomon/actors/dq_solomon_read_actor.cpp +++ b/ydb/library/yql/providers/solomon/actors/dq_solomon_read_actor.cpp @@ -386,7 +386,7 @@ class TDqSolomonReadActor : public NActors::TActorBootstrapped MetricsQueueBatchCountLimit; NCommon::TConfSetting SolomonClientDefaultReplica; NCommon::TConfSetting ComputeActorBatchSize; + NCommon::TConfSetting MaxApiInflight; }; struct TSolomonConfiguration diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp index 14976279eab4..054e33ed1468 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp @@ -343,13 +343,13 @@ class TSolomonDqIntegration: public TDqIntegrationBase { auto& solomonConfig = State_->Configuration; auto& sourceSettings = *source.MutableSettings(); - auto metricsQueuePageSize = solomonConfig->MetricsQueuePageSize.Get().OrElse(5000); + auto metricsQueuePageSize = solomonConfig->MetricsQueuePageSize.Get().OrElse(2000); sourceSettings.insert({"metricsQueuePageSize", ToString(metricsQueuePageSize)}); - auto metricsQueuePrefetchSize = solomonConfig->MetricsQueuePrefetchSize.Get().OrElse(10000); + auto metricsQueuePrefetchSize = solomonConfig->MetricsQueuePrefetchSize.Get().OrElse(4000); sourceSettings.insert({"metricsQueuePrefetchSize", ToString(metricsQueuePrefetchSize)}); - auto metricsQueueBatchCountLimit = solomonConfig->MetricsQueueBatchCountLimit.Get().OrElse(125); + auto metricsQueueBatchCountLimit = solomonConfig->MetricsQueueBatchCountLimit.Get().OrElse(10); sourceSettings.insert({"metricsQueueBatchCountLimit", ToString(metricsQueueBatchCountLimit)}); auto solomonClientDefaultReplica = solomonConfig->SolomonClientDefaultReplica.Get().OrElse(defaultReplica); @@ -358,6 +358,9 @@ class TSolomonDqIntegration: public TDqIntegrationBase { auto computeActorBatchSize = solomonConfig->ComputeActorBatchSize.Get().OrElse(100); sourceSettings.insert({"computeActorBatchSize", ToString(computeActorBatchSize)}); + auto maxApiInflight = solomonConfig->MaxApiInflight.Get().OrElse(40); + sourceSettings.insert({"maxApiInflight", ToString(maxApiInflight)}); + if (!selectors.empty()) { ui64 totalMetricsCount; YQL_ENSURE(TryFromString(settings.TotalMetricsCount(), totalMetricsCount)); diff --git a/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.cpp b/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.cpp index d97cd9970af7..c039c4e1d37a 100644 --- a/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.cpp +++ b/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.cpp @@ -220,6 +220,10 @@ TGetDataResponse ProcessGetDataResponse(NYdbGrpc::TGrpcStatus&& status, ReadResp std::vector timestamps(queryResponse.timestamp_values().values().begin(), queryResponse.timestamp_values().values().end()); std::vector values(queryResponse.double_values().values().begin(), queryResponse.double_values().values().end()); + if (TString name = queryResponse.name()) { + labels["name"] = name; + } + TMetric metric { .Labels = labels, .Type = type, @@ -235,18 +239,20 @@ class TSolomonAccessorClient : public ISolomonAccessorClient, public std::enable public: TSolomonAccessorClient( const TString& defaultReplica, + ui64 maxApiInflight, NYql::NSo::NProto::TDqSolomonSource&& settings, std::shared_ptr credentialsProvider) : DefaultReplica(defaultReplica) + , MaxApiInflight(maxApiInflight) , Settings(std::move(settings)) , CredentialsProvider(credentialsProvider) { - HttpConfig.SetMaxInFlightCount(HttpMaxInflight); + HttpConfig.SetMaxInFlightCount(MaxApiInflight); HttpGateway = IHTTPGateway::Make(&HttpConfig); GrpcConfig.Locator = GetGrpcSolomonEndpoint(); GrpcConfig.EnableSsl = Settings.GetUseSsl(); - GrpcConfig.MaxInFlight = GrpcMaxInflight; + GrpcConfig.MaxInFlight = MaxApiInflight; GrpcClient = std::make_shared(); GrpcConnection = GrpcClient->CreateGRpcServiceConnection(GrpcConfig); } @@ -580,8 +586,7 @@ class TSolomonAccessorClient : public ISolomonAccessorClient, public std::enable private: const TString DefaultReplica; const ui64 ListSizeLimit = 1ull << 20; - const ui64 HttpMaxInflight = 50; - const ui64 GrpcMaxInflight = 50; + const ui64 MaxApiInflight; const NYql::NSo::NProto::TDqSolomonSource Settings; const std::shared_ptr CredentialsProvider; @@ -605,7 +610,12 @@ ISolomonAccessorClient::Make( defaultReplica = it->second; } - return std::make_shared(defaultReplica, std::move(source), credentialsProvider); + ui64 maxApiInflight = 40; + if (auto it = settings.find("maxApiInflight"); it != settings.end()) { + maxApiInflight = FromString(it->second); + } + + return std::make_shared(defaultReplica, maxApiInflight, std::move(source), credentialsProvider); } } // namespace NYql::NSo diff --git a/ydb/library/yql/tools/solomon_emulator_grpc/__main__.py b/ydb/library/yql/tools/solomon_emulator_grpc/__main__.py index 798225687665..57479a7b2f11 100644 --- a/ydb/library/yql/tools/solomon_emulator_grpc/__main__.py +++ b/ydb/library/yql/tools/solomon_emulator_grpc/__main__.py @@ -54,7 +54,6 @@ def Read(self, request: ReadRequest, context) -> ReadResponse: response_query.query_name = "query" timeseries = response_query.timeseries_vector.values.add() - timeseries.name = "name" for key, value in labels.items(): timeseries.labels[key] = str(value) timeseries.type = MetricType.RATE From 9b8309586abdb27f6ed4f7785cf4b753a1c20fcd Mon Sep 17 00:00:00 2001 From: Ivan Sukhov Date: Thu, 31 Jul 2025 15:52:43 +0300 Subject: [PATCH 3/5] Enabling downsampling on time ranges without true points (#20458) --- .../actors/dq_solomon_metrics_queue.cpp | 24 +++++++++----- .../solomon/actors/dq_solomon_read_actor.cpp | 31 +++++++++++++++---- .../solomon/provider/yql_solomon_config.cpp | 1 + .../solomon/provider/yql_solomon_config.h | 1 + .../provider/yql_solomon_dq_integration.cpp | 3 ++ .../provider/yql_solomon_load_meta.cpp | 18 +++++++++-- .../client/solomon_accessor_client.cpp | 24 +++++++------- .../client/solomon_accessor_client.h | 6 ++-- .../opt.yql | 2 +- .../results.txt | 2 +- .../opt.yql | 2 +- .../opt.yql | 2 +- .../results.txt | 2 +- .../opt.yql | 2 +- .../results.txt | 2 +- .../opt.yql | 2 +- .../results.txt | 2 +- .../opt.yql | 2 +- .../results.txt | 2 +- .../opt.yql | 2 +- .../results.txt | 2 +- .../solomon/BadDownsamplingAggregation.sql | 4 +-- .../solomon/BadDownsamplingDisabled.sql | 4 +-- .../suites/solomon/BadDownsamplingFill.sql | 4 +-- .../solomon/BadDownsamplingInterval.sql | 4 +-- .../yql/tests/sql/suites/solomon/Basic.sql | 4 +-- .../suites/solomon/BasicExtractMembers.sql | 4 +-- .../tests/sql/suites/solomon/Downsampling.sql | 4 +-- .../solomon/DownsamplingValidSettings.sql | 4 +-- .../tests/sql/suites/solomon/HistResponse.sql | 4 +-- .../sql/suites/solomon/InvalidProject.sql | 4 +-- .../tests/sql/suites/solomon/LabelColumns.sql | 4 +-- .../yql/tests/sql/suites/solomon/Subquery.sql | 4 +-- .../sql/suites/solomon/UnknownSetting.sql | 4 +-- .../tools/solomon_emulator_grpc/__main__.py | 2 +- .../results.txt | 18 +++++------ .../results.txt | 18 +++++------ .../results.txt | 18 +++++------ .../results.txt | 18 +++++------ .../results.txt | 6 ++-- 40 files changed, 156 insertions(+), 110 deletions(-) diff --git a/ydb/library/yql/providers/solomon/actors/dq_solomon_metrics_queue.cpp b/ydb/library/yql/providers/solomon/actors/dq_solomon_metrics_queue.cpp index de9ed2ff2599..97af31189bbc 100644 --- a/ydb/library/yql/providers/solomon/actors/dq_solomon_metrics_queue.cpp +++ b/ydb/library/yql/providers/solomon/actors/dq_solomon_metrics_queue.cpp @@ -65,17 +65,20 @@ class TDqSolomonMetricsQueueActor : public NActors::TActorBootstrapped credentialsProvider) : CurrentPage(0) , ConsumersCount(consumersCount) + , ReadParams(std::move(readParams)) , PageSize(pageSize) , PrefetchSize(prefetchSize) , BatchCountLimit(batchCountLimit) - , ReadParams(std::move(readParams)) + , TrueRangeFrom(TInstant::Seconds(ReadParams.Source.GetFrom()) - truePointsFindRange) + , TrueRangeTo(TInstant::Seconds(ReadParams.Source.GetTo()) + truePointsFindRange) , CredentialsProvider(credentialsProvider) , SolomonClient(NSo::ISolomonAccessorClient::Make(ReadParams.Source, CredentialsProvider)) {} @@ -277,7 +280,7 @@ class TDqSolomonMetricsQueueActor : public NActors::TActorBootstrapped selectors(ReadParams.Source.GetSelectors().begin(), ReadParams.Source.GetSelectors().end()); ListingFuture = SolomonClient - ->ListMetrics(selectors, PageSize, CurrentPage++) + ->ListMetrics(selectors, TrueRangeFrom, TrueRangeTo, PageSize, CurrentPage++) .Subscribe([actorSystem, selfId = SelfId()]( NThreading::TFuture future) -> void { try { @@ -419,10 +422,12 @@ class TDqSolomonMetricsQueueActor : public NActors::TActorBootstrapped Metrics; TMaybe MaybeIssues; - ui64 PageSize; - ui64 PrefetchSize; - ui64 BatchCountLimit; const TDqSolomonReadParams ReadParams; + const ui64 PageSize; + const ui64 PrefetchSize; + const ui64 BatchCountLimit; + const TInstant TrueRangeFrom; + const TInstant TrueRangeTo; const std::shared_ptr CredentialsProvider; const NSo::ISolomonAccessorClient::TPtr SolomonClient; @@ -455,7 +460,12 @@ NActors::IActor* CreateSolomonMetricsQueueActor( batchCountLimit = FromString(it->second); } - return new TDqSolomonMetricsQueueActor(consumersCount, pageSize, prefetchSize, batchCountLimit, std::move(readParams), credentialsProvider); + ui64 truePointsFindRange = 301; + if (auto it = settings.find("truePointsFindRange"); it != settings.end()) { + truePointsFindRange = FromString(it->second); + } + + return new TDqSolomonMetricsQueueActor(consumersCount, std::move(readParams), pageSize, prefetchSize, batchCountLimit, TDuration::Seconds(truePointsFindRange), credentialsProvider); } } // namespace NYql::NDq diff --git a/ydb/library/yql/providers/solomon/actors/dq_solomon_read_actor.cpp b/ydb/library/yql/providers/solomon/actors/dq_solomon_read_actor.cpp index 3d4e6df214c9..b60b298860ff 100644 --- a/ydb/library/yql/providers/solomon/actors/dq_solomon_read_actor.cpp +++ b/ydb/library/yql/providers/solomon/actors/dq_solomon_read_actor.cpp @@ -80,6 +80,7 @@ class TDqSolomonReadActor : public NActors::TActorBootstrapped> SplitTimeIntervalIntoRanges(TInstant from, TInstant to, ui64 pointsCount) const { + std::vector> SplitTimeIntervalIntoRanges(ui64 pointsCount) const { + TInstant from = TrueRangeFrom; + TInstant to = TrueRangeTo; + std::vector> result; if (pointsCount == 0) { return result; @@ -552,6 +563,8 @@ class TDqSolomonReadActor : public NActors::TActorBootstrapped::TPtr RetryPolicy; @@ -618,6 +631,11 @@ std::pair CreateDqSolom computeActorBatchSize = FromString(it->second); } + ui64 truePointsFindRange = 301; + if (auto it = settings.find("truePointsFindRange"); it != settings.end()) { + truePointsFindRange = FromString(it->second); + } + auto credentialsProviderFactory = CreateCredentialsProviderFactoryForStructuredToken(credentialsFactory, token); auto credentialsProvider = credentialsProviderFactory->CreateProvider(); @@ -630,6 +648,7 @@ std::pair CreateDqSolom programBuilder, std::move(params), computeActorBatchSize, + TDuration::Seconds(truePointsFindRange), metricsQueueConsumersCountDelta, metricsQueueActor, counters, diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_config.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_config.cpp index a4184097e09f..ec55f37bed87 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_config.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_config.cpp @@ -8,6 +8,7 @@ TSolomonConfiguration::TSolomonConfiguration() { REGISTER_SETTING(*this, _EnableReading); REGISTER_SETTING(*this, _EnableRuntimeListing); + REGISTER_SETTING(*this, _TruePointsFindRange); REGISTER_SETTING(*this, MetricsQueuePageSize); REGISTER_SETTING(*this, MetricsQueuePrefetchSize); REGISTER_SETTING(*this, MetricsQueueBatchCountLimit); diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h b/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h index 9aef900f0b65..26f172d2555e 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h @@ -12,6 +12,7 @@ struct TSolomonSettings { NCommon::TConfSetting _EnableReading; NCommon::TConfSetting _EnableRuntimeListing; + NCommon::TConfSetting _TruePointsFindRange; NCommon::TConfSetting MetricsQueuePageSize; NCommon::TConfSetting MetricsQueuePrefetchSize; NCommon::TConfSetting MetricsQueueBatchCountLimit; diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp index 054e33ed1468..a71e49b7e364 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp @@ -358,6 +358,9 @@ class TSolomonDqIntegration: public TDqIntegrationBase { auto computeActorBatchSize = solomonConfig->ComputeActorBatchSize.Get().OrElse(100); sourceSettings.insert({"computeActorBatchSize", ToString(computeActorBatchSize)}); + auto truePointsFindRange = solomonConfig->_TruePointsFindRange.Get().OrElse(301); + sourceSettings.insert({"truePointsFindRange", ToString(truePointsFindRange)}); + auto maxApiInflight = solomonConfig->MaxApiInflight.Get().OrElse(40); sourceSettings.insert({"maxApiInflight", ToString(maxApiInflight)}); diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_load_meta.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_load_meta.cpp index 0da9a4ced745..847e3b2836ad 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_load_meta.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_load_meta.cpp @@ -66,6 +66,20 @@ class TSolomonLoadTableMetadataTransformer : public TGraphTransformerBase { if (auto maybeSelectors = ExtractSetting(settings, "selectors")) { NSo::NProto::TDqSolomonSource source = NSo::FillSolomonSource(clusterDesc, soReadObject.Object().Project().StringValue()); + + TInstant from; + if (auto time = ExtractSetting(settings, "from")) { + from = TInstant::ParseIso8601(*time); + } else { + from = TInstant::Now() - TDuration::Days(7); + } + + TInstant to; + if (auto time = ExtractSetting(settings, "to")) { + to = TInstant::ParseIso8601(*time); + } else { + to = TInstant::Now(); + } auto selectors = NSo::ExtractSelectorValues(*maybeSelectors); if (source.GetClusterType() == NSo::NProto::CT_MONITORING) { @@ -83,8 +97,8 @@ class TSolomonLoadTableMetadataTransformer : public TGraphTransformerBase { auto credentialsProvider = providerFactory->CreateProvider(); auto solomonClient = NSo::ISolomonAccessorClient::Make(std::move(source), credentialsProvider); - auto labelNamesFuture = solomonClient->GetLabelNames(selectors); - auto listMetricsFuture = solomonClient->ListMetrics(selectors, 30, 0); + auto labelNamesFuture = solomonClient->GetLabelNames(selectors, from, to); + auto listMetricsFuture = solomonClient->ListMetrics(selectors, from, to, 30, 0); LabelNamesRequests_[soReadObject.Raw()] = { .SolomonClient = solomonClient, diff --git a/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.cpp b/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.cpp index c039c4e1d37a..af08c07f11f5 100644 --- a/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.cpp +++ b/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.cpp @@ -262,8 +262,8 @@ class TSolomonAccessorClient : public ISolomonAccessorClient, public std::enable } public: - NThreading::TFuture GetLabelNames(const std::map& selectors) const override final { - auto requestUrl = BuildGetLabelsUrl(selectors); + NThreading::TFuture GetLabelNames(const std::map& selectors, TInstant from, TInstant to) const override final { + auto requestUrl = BuildGetLabelsUrl(selectors, from, to); auto resultPromise = NThreading::NewPromise(); @@ -279,8 +279,8 @@ class TSolomonAccessorClient : public ISolomonAccessorClient, public std::enable return resultPromise.GetFuture(); } - NThreading::TFuture ListMetrics(const std::map& selectors, int pageSize, int page) const override final { - auto requestUrl = BuildListMetricsUrl(selectors, pageSize, page); + NThreading::TFuture ListMetrics(const std::map& selectors, TInstant from, TInstant to, int pageSize, int page) const override final { + auto requestUrl = BuildListMetricsUrl(selectors, from, to, pageSize, page); auto resultPromise = NThreading::NewPromise(); @@ -296,11 +296,9 @@ class TSolomonAccessorClient : public ISolomonAccessorClient, public std::enable return resultPromise.GetFuture(); } - NThreading::TFuture GetPointsCount(const std::map& selectors) const override final { + NThreading::TFuture GetPointsCount(const std::map& selectors, TInstant from, TInstant to) const override final { auto resultPromise = NThreading::NewPromise(); - TInstant from = TInstant::Seconds(Settings.GetFrom()); - TInstant to = TInstant::Seconds(Settings.GetTo()); TInstant sevenDaysAgo = TInstant::Now() - TDuration::Days(7); // points older then a week ago are automatically downsampled by solomon backend TInstant downsamplingFrom = from; @@ -453,7 +451,7 @@ class TSolomonAccessorClient : public ISolomonAccessorClient, public std::enable } } - TString BuildGetLabelsUrl(const std::map& selectors) const { + TString BuildGetLabelsUrl(const std::map& selectors, TInstant from, TInstant to) const { TUrlBuilder builder(GetHttpSolomonEndpoint()); builder.AddPathComponent("api"); @@ -465,13 +463,13 @@ class TSolomonAccessorClient : public ISolomonAccessorClient, public std::enable builder.AddUrlParam("selectors", BuildSelectorsProgram(selectors)); builder.AddUrlParam("forceCluster", DefaultReplica); - builder.AddUrlParam("from", TInstant::Seconds(Settings.GetFrom()).ToString()); - builder.AddUrlParam("to", TInstant::Seconds(Settings.GetTo()).ToString()); + builder.AddUrlParam("from", from.ToString()); + builder.AddUrlParam("to", to.ToString()); return builder.Build(); } - TString BuildListMetricsUrl(const std::map& selectors, int pageSize, int page) const { + TString BuildListMetricsUrl(const std::map& selectors, TInstant from, TInstant to, int pageSize, int page) const { TUrlBuilder builder(GetHttpSolomonEndpoint()); builder.AddPathComponent("api"); @@ -482,8 +480,8 @@ class TSolomonAccessorClient : public ISolomonAccessorClient, public std::enable builder.AddUrlParam("selectors", BuildSelectorsProgram(selectors)); builder.AddUrlParam("forceCluster", DefaultReplica); - builder.AddUrlParam("from", TInstant::Seconds(Settings.GetFrom()).ToString()); - builder.AddUrlParam("to", TInstant::Seconds(Settings.GetTo()).ToString()); + builder.AddUrlParam("from", from.ToString()); + builder.AddUrlParam("to", to.ToString()); builder.AddUrlParam("pageSize", std::to_string(pageSize)); builder.AddUrlParam("page", std::to_string(page)); diff --git a/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.h b/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.h index c5bbd5cdefdd..14540fc70c43 100644 --- a/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.h +++ b/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.h @@ -19,9 +19,9 @@ class ISolomonAccessorClient { std::shared_ptr credentialsProvider); public: - virtual NThreading::TFuture GetLabelNames(const std::map& selectors) const = 0; - virtual NThreading::TFuture ListMetrics(const std::map& selectors, int pageSize, int page) const = 0; - virtual NThreading::TFuture GetPointsCount(const std::map& selectors) const = 0; + virtual NThreading::TFuture GetLabelNames(const std::map& selectors, TInstant from, TInstant to) const = 0; + virtual NThreading::TFuture ListMetrics(const std::map& selectors, TInstant from, TInstant to, int pageSize, int page) const = 0; + virtual NThreading::TFuture GetPointsCount(const std::map& selectors, TInstant from, TInstant to) const = 0; virtual NThreading::TFuture GetData(const std::map& selectors, TInstant from, TInstant to) const = 0; virtual NThreading::TFuture GetData(TString selectors, TInstant from, TInstant to) const = 0; }; diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Basic-default.txt_/opt.yql b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Basic-default.txt_/opt.yql index f8b0dd5be37a..d79f0660e437 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Basic-default.txt_/opt.yql +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Basic-default.txt_/opt.yql @@ -4,7 +4,7 @@ (let $3 (DataType 'String)) (let $4 (StructType '('"labels" (DictType $3 $3)) '('"ts" (DataType 'Datetime)) '('type $3) '('"value" (OptionalType (DataType 'Double))))) (let $5 '('"labels" '"value" '"ts" 'type)) -(let $6 (SoSourceSettings world '"my_project" (SecureParam '"cluster:default_local_solomon") $4 $5 '() '"2023-12-08T14:40:39Z" '"2023-12-08T14:45:39Z" '"" '"{}" (Bool '"false") '"AVG" '"PREVIOUS" (Uint32 '"15") '() '"")) +(let $6 (SoSourceSettings world '"my_project" (SecureParam '"cluster:default_local_solomon") $4 $5 '() '"1970-01-01T00:00:01Z" '"1970-01-02T00:00:01Z" '"" '"{}" (Bool '"false") '"AVG" '"PREVIOUS" (Uint32 '"15") '() '"")) (let $7 (DqStage '((DqSource (DataSource '"solomon" '"local_solomon") $6)) (lambda '($10) $10) '('('"_logical_id" '0)))) (let $8 (DqStage '((DqCnUnionAll (TDqOutput $7 '"0"))) (lambda '($11) $11) '('('"_logical_id" '0)))) (let $9 (ResPull! $1 $2 (Key) (DqCnResult (TDqOutput $8 '"0") '()) '('('type) '('autoref)) '"dq")) diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Basic-default.txt_/results.txt b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Basic-default.txt_/results.txt index 59ad016e435f..5dac7cfc5302 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Basic-default.txt_/results.txt +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Basic-default.txt_/results.txt @@ -1 +1 @@ -[{"Write"=[{"Type"=["ListType";["StructType";[["labels";["DictType";["DataType";"String"];["DataType";"String"]]];["ts";["DataType";"Datetime"]];["type";["DataType";"String"]];["value";["OptionalType";["DataType";"Double"]]]]]];"Data"=[[[["downsampling.aggregation";"4"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";["100"]];[[["downsampling.aggregation";"4"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";["200"]];[[["downsampling.aggregation";"4"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";["300"]]]}]}] \ No newline at end of file +[{"Write"=[{"Type"=["ListType";["StructType";[["labels";["DictType";["DataType";"String"];["DataType";"String"]]];["ts";["DataType";"Datetime"]];["type";["DataType";"String"]];["value";["OptionalType";["DataType";"Double"]]]]]];"Data"=[[[["downsampling.aggregation";"4"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 86401\n"]];"10";"RATE";["100"]];[[["downsampling.aggregation";"4"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 86401\n"]];"20";"RATE";["200"]];[[["downsampling.aggregation";"4"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 86401\n"]];"30";"RATE";["300"]]]}]}] \ No newline at end of file diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-BasicExtractMembers-default.txt_/opt.yql b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-BasicExtractMembers-default.txt_/opt.yql index d87bac8a5e6a..2e86037391c0 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-BasicExtractMembers-default.txt_/opt.yql +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-BasicExtractMembers-default.txt_/opt.yql @@ -2,7 +2,7 @@ (let $1 (Configure! world (DataSource '"config") '"DqEngine" '"force")) (let $2 (DataSink 'result)) (let $3 '('"value")) -(let $4 (SoSourceSettings world '"my_project" (SecureParam '"cluster:default_local_solomon") (StructType '('"value" (OptionalType (DataType 'Double)))) $3 '() '"2023-12-08T14:40:39Z" '"2023-12-08T14:45:39Z" '"" '"{}" (Bool '"false") '"AVG" '"PREVIOUS" (Uint32 '"15") '() '"")) +(let $4 (SoSourceSettings world '"my_project" (SecureParam '"cluster:default_local_solomon") (StructType '('"value" (OptionalType (DataType 'Double)))) $3 '() '"1970-01-01T00:00:01Z" '"1970-01-02T00:00:01Z" '"" '"{}" (Bool '"false") '"AVG" '"PREVIOUS" (Uint32 '"15") '() '"")) (let $5 (DqStage '((DqSource (DataSource '"solomon" '"local_solomon") $4)) (lambda '($9) $9) '('('"_logical_id" '0)))) (let $6 (DqStage '((DqCnUnionAll (TDqOutput $5 '"0"))) (lambda '($10) $10) '('('"_logical_id" '0)))) (let $7 '('('type) '('autoref) '('columns $3))) diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Downsampling-default.txt_/opt.yql b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Downsampling-default.txt_/opt.yql index 052e1c806b02..785e0de42ac9 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Downsampling-default.txt_/opt.yql +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Downsampling-default.txt_/opt.yql @@ -4,7 +4,7 @@ (let $3 (DataType 'String)) (let $4 (StructType '('"labels" (DictType $3 $3)) '('"ts" (DataType 'Datetime)) '('type $3) '('"value" (OptionalType (DataType 'Double))))) (let $5 '('"labels" '"value" '"ts" 'type)) -(let $6 (SoSourceSettings world '"my_project" (SecureParam '"cluster:default_local_solomon") $4 $5 '() '"2023-12-08T14:40:39Z" '"2023-12-08T14:45:39Z" '"" '"{}" (Bool '"false") '"SUM" '"PREVIOUS" (Uint32 '"25") '() '"")) +(let $6 (SoSourceSettings world '"my_project" (SecureParam '"cluster:default_local_solomon") $4 $5 '() '"1970-01-01T00:00:01Z" '"1970-01-02T00:00:01Z" '"" '"{}" (Bool '"false") '"SUM" '"PREVIOUS" (Uint32 '"25") '() '"")) (let $7 (DqStage '((DqSource (DataSource '"solomon" '"local_solomon") $6)) (lambda '($10) $10) '('('"_logical_id" '0)))) (let $8 (DqStage '((DqCnUnionAll (TDqOutput $7 '"0"))) (lambda '($11) $11) '('('"_logical_id" '0)))) (let $9 (ResPull! $1 $2 (Key) (DqCnResult (TDqOutput $8 '"0") '()) '('('type) '('autoref)) '"dq")) diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Downsampling-default.txt_/results.txt b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Downsampling-default.txt_/results.txt index 9defd3e8b2d4..01e3b658838a 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Downsampling-default.txt_/results.txt +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Downsampling-default.txt_/results.txt @@ -1 +1 @@ -[{"Write"=[{"Type"=["ListType";["StructType";[["labels";["DictType";["DataType";"String"];["DataType";"String"]]];["ts";["DataType";"Datetime"]];["type";["DataType";"String"]];["value";["OptionalType";["DataType";"Double"]]]]]];"Data"=[[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 25000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";["100"]];[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 25000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";["200"]];[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 25000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";["300"]]]}]}] \ No newline at end of file +[{"Write"=[{"Type"=["ListType";["StructType";[["labels";["DictType";["DataType";"String"];["DataType";"String"]]];["ts";["DataType";"Datetime"]];["type";["DataType";"String"]];["value";["OptionalType";["DataType";"Double"]]]]]];"Data"=[[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 25000"];["from";"seconds: 1\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 86401\n"]];"10";"RATE";["100"]];[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 25000"];["from";"seconds: 1\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 86401\n"]];"20";"RATE";["200"]];[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 25000"];["from";"seconds: 1\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 86401\n"]];"30";"RATE";["300"]]]}]}] \ No newline at end of file diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-DownsamplingValidSettings-default.txt_/opt.yql b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-DownsamplingValidSettings-default.txt_/opt.yql index 25aba09ac72e..372609f24f4b 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-DownsamplingValidSettings-default.txt_/opt.yql +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-DownsamplingValidSettings-default.txt_/opt.yql @@ -4,7 +4,7 @@ (let $3 (DataType 'String)) (let $4 (StructType '('"labels" (DictType $3 $3)) '('"ts" (DataType 'Datetime)) '('type $3) '('"value" (OptionalType (DataType 'Double))))) (let $5 '('"labels" '"value" '"ts" 'type)) -(let $6 (SoSourceSettings world '"my_project" (SecureParam '"cluster:default_local_solomon") $4 $5 '() '"2023-12-08T14:40:39Z" '"2023-12-08T14:45:39Z" '"" '"{}" (Bool '"false") '"SUM" '"PREVIOUS" (Uint32 '"15") '() '"")) +(let $6 (SoSourceSettings world '"my_project" (SecureParam '"cluster:default_local_solomon") $4 $5 '() '"1970-01-01T00:00:01Z" '"1970-01-02T00:00:01Z" '"" '"{}" (Bool '"false") '"SUM" '"PREVIOUS" (Uint32 '"15") '() '"")) (let $7 (DqStage '((DqSource (DataSource '"solomon" '"local_solomon") $6)) (lambda '($10) $10) '('('"_logical_id" '0)))) (let $8 (DqStage '((DqCnUnionAll (TDqOutput $7 '"0"))) (lambda '($11) $11) '('('"_logical_id" '0)))) (let $9 (ResPull! $1 $2 (Key) (DqCnResult (TDqOutput $8 '"0") '()) '('('type) '('autoref)) '"dq")) diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-DownsamplingValidSettings-default.txt_/results.txt b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-DownsamplingValidSettings-default.txt_/results.txt index a616aa78844f..d0c62a752d9a 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-DownsamplingValidSettings-default.txt_/results.txt +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-DownsamplingValidSettings-default.txt_/results.txt @@ -1 +1 @@ -[{"Write"=[{"Type"=["ListType";["StructType";[["labels";["DictType";["DataType";"String"];["DataType";"String"]]];["ts";["DataType";"Datetime"]];["type";["DataType";"String"]];["value";["OptionalType";["DataType";"Double"]]]]]];"Data"=[[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";["100"]];[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";["200"]];[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";["300"]]]}]}] \ No newline at end of file +[{"Write"=[{"Type"=["ListType";["StructType";[["labels";["DictType";["DataType";"String"];["DataType";"String"]]];["ts";["DataType";"Datetime"]];["type";["DataType";"String"]];["value";["OptionalType";["DataType";"Double"]]]]]];"Data"=[[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 86401\n"]];"10";"RATE";["100"]];[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 86401\n"]];"20";"RATE";["200"]];[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 86401\n"]];"30";"RATE";["300"]]]}]}] \ No newline at end of file diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-HistResponse-default.txt_/opt.yql b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-HistResponse-default.txt_/opt.yql index 533b5863afbb..a791d053cc76 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-HistResponse-default.txt_/opt.yql +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-HistResponse-default.txt_/opt.yql @@ -4,7 +4,7 @@ (let $3 (DataType 'String)) (let $4 (StructType '('"labels" (DictType $3 $3)) '('"ts" (DataType 'Datetime)) '('type $3) '('"value" (OptionalType (DataType 'Double))))) (let $5 '('"labels" '"value" '"ts" 'type)) -(let $6 (SoSourceSettings world '"hist" (SecureParam '"cluster:default_local_solomon") $4 $5 '() '"2023-12-08T14:40:39Z" '"2023-12-08T14:45:39Z" '"" '"histogram_percentile(95, {})" (Bool '"false") '"AVG" '"PREVIOUS" (Uint32 '"15") '() '"")) +(let $6 (SoSourceSettings world '"hist" (SecureParam '"cluster:default_local_solomon") $4 $5 '() '"1970-01-01T00:00:01Z" '"1970-01-02T00:00:01Z" '"" '"histogram_percentile(95, {})" (Bool '"false") '"AVG" '"PREVIOUS" (Uint32 '"15") '() '"")) (let $7 (DqStage '((DqSource (DataSource '"solomon" '"local_solomon") $6)) (lambda '($10) $10) '('('"_logical_id" '0)))) (let $8 (DqStage '((DqCnUnionAll (TDqOutput $7 '"0"))) (lambda '($11) $11) '('('"_logical_id" '0)))) (let $9 (ResPull! $1 $2 (Key) (DqCnResult (TDqOutput $8 '"0") '()) '('('type) '('autoref)) '"dq")) diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-HistResponse-default.txt_/results.txt b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-HistResponse-default.txt_/results.txt index 568abc9e6d8f..87dfece407aa 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-HistResponse-default.txt_/results.txt +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-HistResponse-default.txt_/results.txt @@ -1 +1 @@ -[{"Write"=[{"Type"=["ListType";["StructType";[["labels";["DictType";["DataType";"String"];["DataType";"String"]]];["ts";["DataType";"Datetime"]];["type";["DataType";"String"]];["value";["OptionalType";["DataType";"Double"]]]]]];"Data"=[[[["downsampling.aggregation";"4"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 28"];["project";"hist"];["to";"seconds: 1702046739\n"]];"0";"RATE";["100"]];[[["downsampling.aggregation";"4"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 28"];["project";"hist"];["to";"seconds: 1702046739\n"]];"0";"RATE";["200"]];[[["downsampling.aggregation";"4"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 28"];["project";"hist"];["to";"seconds: 1702046739\n"]];"0";"RATE";["300"]]]}]}] \ No newline at end of file +[{"Write"=[{"Type"=["ListType";["StructType";[["labels";["DictType";["DataType";"String"];["DataType";"String"]]];["ts";["DataType";"Datetime"]];["type";["DataType";"String"]];["value";["OptionalType";["DataType";"Double"]]]]]];"Data"=[[[["downsampling.aggregation";"4"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1\n"];["downsampling.disabled";"bool False"];["program";"program length 28"];["project";"hist"];["to";"seconds: 86401\n"]];"10";"RATE";["100"]];[[["downsampling.aggregation";"4"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1\n"];["downsampling.disabled";"bool False"];["program";"program length 28"];["project";"hist"];["to";"seconds: 86401\n"]];"20";"RATE";["200"]];[[["downsampling.aggregation";"4"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1\n"];["downsampling.disabled";"bool False"];["program";"program length 28"];["project";"hist"];["to";"seconds: 86401\n"]];"30";"RATE";["300"]]]}]}] \ No newline at end of file diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-LabelColumns-default.txt_/opt.yql b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-LabelColumns-default.txt_/opt.yql index 6610d0a20d12..c0428ab6ec95 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-LabelColumns-default.txt_/opt.yql +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-LabelColumns-default.txt_/opt.yql @@ -8,7 +8,7 @@ (let $7 (StructType '($3 $4) '($5 $4) '($6 $4) '('"project" $4) '('"ts" (DataType 'Datetime)) '('type $4) '('"value" (OptionalType (DataType 'Double))))) (let $8 '('"value" '"ts" 'type)) (let $9 '($3 $5 '"project" $6)) -(let $10 (SoSourceSettings world '"my_project" (SecureParam '"cluster:default_local_solomon") $7 $8 $9 '"2023-12-08T14:40:39Z" '"2023-12-08T14:45:39Z" '"" '"{}" (Bool '"false") '"AVG" '"PREVIOUS" (Uint32 '"15") '() '"")) +(let $10 (SoSourceSettings world '"my_project" (SecureParam '"cluster:default_local_solomon") $7 $8 $9 '"1970-01-01T00:00:01Z" '"1970-01-02T00:00:01Z" '"" '"{}" (Bool '"false") '"AVG" '"PREVIOUS" (Uint32 '"15") '() '"")) (let $11 (DqStage '((DqSource (DataSource '"solomon" '"local_solomon") $10)) (lambda '($14) $14) '('('"_logical_id" '0)))) (let $12 (DqStage '((DqCnUnionAll (TDqOutput $11 '"0"))) (lambda '($15) $15) '('('"_logical_id" '0)))) (let $13 (ResPull! $1 $2 (Key) (DqCnResult (TDqOutput $12 '"0") '()) '('('type) '('autoref)) '"dq")) diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-LabelColumns-default.txt_/results.txt b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-LabelColumns-default.txt_/results.txt index a3886d17a7f5..7424e4690bd5 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-LabelColumns-default.txt_/results.txt +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-LabelColumns-default.txt_/results.txt @@ -1 +1 @@ -[{"Write"=[{"Type"=["ListType";["StructType";[["downsampling.disabled";["DataType";"String"]];["downsampling.fill";["DataType";"String"]];["downsampling.gridMillis";["DataType";"String"]];["project";["DataType";"String"]];["ts";["DataType";"Datetime"]];["type";["DataType";"String"]];["value";["OptionalType";["DataType";"Double"]]]]]];"Data"=[["bool False";"3";"int 15000";"my_project";"0";"RATE";["100"]];["bool False";"3";"int 15000";"my_project";"0";"RATE";["200"]];["bool False";"3";"int 15000";"my_project";"0";"RATE";["300"]]]}]}] \ No newline at end of file +[{"Write"=[{"Type"=["ListType";["StructType";[["downsampling.disabled";["DataType";"String"]];["downsampling.fill";["DataType";"String"]];["downsampling.gridMillis";["DataType";"String"]];["project";["DataType";"String"]];["ts";["DataType";"Datetime"]];["type";["DataType";"String"]];["value";["OptionalType";["DataType";"Double"]]]]]];"Data"=[["bool False";"3";"int 15000";"my_project";"10";"RATE";["100"]];["bool False";"3";"int 15000";"my_project";"20";"RATE";["200"]];["bool False";"3";"int 15000";"my_project";"30";"RATE";["300"]]]}]}] \ No newline at end of file diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Subquery-default.txt_/opt.yql b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Subquery-default.txt_/opt.yql index 047992dc9288..1bd756e5779b 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Subquery-default.txt_/opt.yql +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Subquery-default.txt_/opt.yql @@ -4,7 +4,7 @@ (let $3 (DataType 'String)) (let $4 (StructType '('"labels" (DictType $3 $3)) '('"ts" (DataType 'Datetime)) '('type $3) '('"value" (OptionalType (DataType 'Double))))) (let $5 '('"labels" '"value" '"ts" 'type)) -(let $6 (SoSourceSettings world '"my_project" (SecureParam '"cluster:default_local_solomon") $4 $5 '() '"2023-12-08T14:40:39Z" '"2023-12-08T14:45:39Z" '"" '"{}" (Bool '"false") '"SUM" '"PREVIOUS" (Uint32 '"25") '() '"")) +(let $6 (SoSourceSettings world '"my_project" (SecureParam '"cluster:default_local_solomon") $4 $5 '() '"1970-01-01T00:00:01Z" '"1970-01-02T00:00:01Z" '"" '"{}" (Bool '"false") '"SUM" '"PREVIOUS" (Uint32 '"25") '() '"")) (let $7 (DqStage '((DqSource (DataSource '"solomon" '"local_solomon") $6)) (lambda '($11) $11) '('('"_logical_id" '0)))) (let $8 (DqStage '((DqCnUnionAll (TDqOutput $7 '"0"))) (lambda '($12) $12) '('('"_logical_id" '0)))) (let $9 '('('type) '('autoref) '('unordered))) diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Subquery-default.txt_/results.txt b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Subquery-default.txt_/results.txt index a8106499556e..3cd6fa077413 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Subquery-default.txt_/results.txt +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Subquery-default.txt_/results.txt @@ -1 +1 @@ -[{"Write"=[{"Type"=["ListType";["StructType";[["labels";["DictType";["DataType";"String"];["DataType";"String"]]];["ts";["DataType";"Datetime"]];["type";["DataType";"String"]];["value";["OptionalType";["DataType";"Double"]]]]]];"Data"=[[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 25000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";["100"]];[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 25000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";["200"]];[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 25000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";["300"]]]}];"Unordered"=%true}] \ No newline at end of file +[{"Write"=[{"Type"=["ListType";["StructType";[["labels";["DictType";["DataType";"String"];["DataType";"String"]]];["ts";["DataType";"Datetime"]];["type";["DataType";"String"]];["value";["OptionalType";["DataType";"Double"]]]]]];"Data"=[[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 25000"];["from";"seconds: 1\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 86401\n"]];"10";"RATE";["100"]];[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 25000"];["from";"seconds: 1\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 86401\n"]];"20";"RATE";["200"]];[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 25000"];["from";"seconds: 1\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 86401\n"]];"30";"RATE";["300"]]]}];"Unordered"=%true}] \ No newline at end of file diff --git a/ydb/library/yql/tests/sql/suites/solomon/BadDownsamplingAggregation.sql b/ydb/library/yql/tests/sql/suites/solomon/BadDownsamplingAggregation.sql index dda21f9ce617..8e8f3cf58e18 100644 --- a/ydb/library/yql/tests/sql/suites/solomon/BadDownsamplingAggregation.sql +++ b/ydb/library/yql/tests/sql/suites/solomon/BadDownsamplingAggregation.sql @@ -1,6 +1,6 @@ SELECT * FROM local_solomon.my_project WITH ( program = @@{}@@, - from = "2023-12-08T14:40:39Z", - to = "2023-12-08T14:45:39Z", + from = "1970-01-01T00:00:01Z", + to = "1970-01-02T00:00:01Z", `downsampling.aggregation` = "ABC" ); diff --git a/ydb/library/yql/tests/sql/suites/solomon/BadDownsamplingDisabled.sql b/ydb/library/yql/tests/sql/suites/solomon/BadDownsamplingDisabled.sql index 61ca6e90b5bb..9300a6e4f905 100644 --- a/ydb/library/yql/tests/sql/suites/solomon/BadDownsamplingDisabled.sql +++ b/ydb/library/yql/tests/sql/suites/solomon/BadDownsamplingDisabled.sql @@ -1,6 +1,6 @@ SELECT * FROM local_solomon.my_project WITH ( program = @@{}@@, - from = "2023-12-08T14:40:39Z", - to = "2023-12-08T14:45:39Z", + from = "1970-01-01T00:00:01Z", + to = "1970-01-02T00:00:01Z", `downsampling.disabled` = "ABC" ); diff --git a/ydb/library/yql/tests/sql/suites/solomon/BadDownsamplingFill.sql b/ydb/library/yql/tests/sql/suites/solomon/BadDownsamplingFill.sql index cf7dac7f357f..f8e5d5e654d4 100644 --- a/ydb/library/yql/tests/sql/suites/solomon/BadDownsamplingFill.sql +++ b/ydb/library/yql/tests/sql/suites/solomon/BadDownsamplingFill.sql @@ -1,6 +1,6 @@ SELECT * FROM local_solomon.my_project WITH ( program = @@{}@@, - from = "2023-12-08T14:40:39Z", - to = "2023-12-08T14:45:39Z", + from = "1970-01-01T00:00:01Z", + to = "1970-01-02T00:00:01Z", `downsampling.fill` = "ABC" ); diff --git a/ydb/library/yql/tests/sql/suites/solomon/BadDownsamplingInterval.sql b/ydb/library/yql/tests/sql/suites/solomon/BadDownsamplingInterval.sql index e5f24c16e968..f5452d69df1d 100644 --- a/ydb/library/yql/tests/sql/suites/solomon/BadDownsamplingInterval.sql +++ b/ydb/library/yql/tests/sql/suites/solomon/BadDownsamplingInterval.sql @@ -1,6 +1,6 @@ SELECT * FROM local_solomon.my_project WITH ( program = @@{}@@, - from = "2023-12-08T14:40:39Z", - to = "2023-12-08T14:45:39Z", + from = "1970-01-01T00:00:01Z", + to = "1970-01-02T00:00:01Z", `downsampling.grid_interval` = "ABC" ); diff --git a/ydb/library/yql/tests/sql/suites/solomon/Basic.sql b/ydb/library/yql/tests/sql/suites/solomon/Basic.sql index d59089121f96..c9d6ab7602cd 100644 --- a/ydb/library/yql/tests/sql/suites/solomon/Basic.sql +++ b/ydb/library/yql/tests/sql/suites/solomon/Basic.sql @@ -1,5 +1,5 @@ SELECT * FROM local_solomon.my_project WITH ( program = @@{}@@, - from = "2023-12-08T14:40:39Z", - to = "2023-12-08T14:45:39Z" + from = "1970-01-01T00:00:01Z", + to = "1970-01-02T00:00:01Z" ); diff --git a/ydb/library/yql/tests/sql/suites/solomon/BasicExtractMembers.sql b/ydb/library/yql/tests/sql/suites/solomon/BasicExtractMembers.sql index 2e1fb26903d2..027ad92e2287 100644 --- a/ydb/library/yql/tests/sql/suites/solomon/BasicExtractMembers.sql +++ b/ydb/library/yql/tests/sql/suites/solomon/BasicExtractMembers.sql @@ -1,5 +1,5 @@ SELECT value FROM local_solomon.my_project WITH ( program = @@{}@@, - from = "2023-12-08T14:40:39Z", - to = "2023-12-08T14:45:39Z" + from = "1970-01-01T00:00:01Z", + to = "1970-01-02T00:00:01Z" ); diff --git a/ydb/library/yql/tests/sql/suites/solomon/Downsampling.sql b/ydb/library/yql/tests/sql/suites/solomon/Downsampling.sql index a960778384a8..92c8fb574d7a 100644 --- a/ydb/library/yql/tests/sql/suites/solomon/Downsampling.sql +++ b/ydb/library/yql/tests/sql/suites/solomon/Downsampling.sql @@ -1,7 +1,7 @@ SELECT * FROM local_solomon.my_project WITH ( program = @@{}@@, - from = "2023-12-08T14:40:39Z", - to = "2023-12-08T14:45:39Z", + from = "1970-01-01T00:00:01Z", + to = "1970-01-02T00:00:01Z", `downsampling.aggregation` = "SUM", `downsampling.grid_interval` = "25", `downsampling.fill` = "PREVIOUS" diff --git a/ydb/library/yql/tests/sql/suites/solomon/DownsamplingValidSettings.sql b/ydb/library/yql/tests/sql/suites/solomon/DownsamplingValidSettings.sql index c1f59e9de9d7..cd954189ba71 100644 --- a/ydb/library/yql/tests/sql/suites/solomon/DownsamplingValidSettings.sql +++ b/ydb/library/yql/tests/sql/suites/solomon/DownsamplingValidSettings.sql @@ -1,7 +1,7 @@ SELECT * FROM local_solomon.my_project WITH ( program = @@{}@@, - from = "2023-12-08T14:40:39Z", - to = "2023-12-08T14:45:39Z", + from = "1970-01-01T00:00:01Z", + to = "1970-01-02T00:00:01Z", `downsampling.aggregation` = "AVG", `downsampling.aggregation` = "COUNT", diff --git a/ydb/library/yql/tests/sql/suites/solomon/HistResponse.sql b/ydb/library/yql/tests/sql/suites/solomon/HistResponse.sql index 762da54aab62..d613193918a6 100644 --- a/ydb/library/yql/tests/sql/suites/solomon/HistResponse.sql +++ b/ydb/library/yql/tests/sql/suites/solomon/HistResponse.sql @@ -1,5 +1,5 @@ SELECT * FROM local_solomon.hist WITH ( program = @@histogram_percentile(95, {})@@, - from = "2023-12-08T14:40:39Z", - to = "2023-12-08T14:45:39Z" + from = "1970-01-01T00:00:01Z", + to = "1970-01-02T00:00:01Z" ); diff --git a/ydb/library/yql/tests/sql/suites/solomon/InvalidProject.sql b/ydb/library/yql/tests/sql/suites/solomon/InvalidProject.sql index a9275c07d1a5..060b2396405a 100644 --- a/ydb/library/yql/tests/sql/suites/solomon/InvalidProject.sql +++ b/ydb/library/yql/tests/sql/suites/solomon/InvalidProject.sql @@ -1,5 +1,5 @@ SELECT * FROM local_solomon.invalid WITH ( program = @@{}@@, - from = "2023-12-08T14:40:39Z", - to = "2023-12-08T14:45:39Z" + from = "1970-01-01T00:00:01Z", + to = "1970-01-02T00:00:01Z" ); diff --git a/ydb/library/yql/tests/sql/suites/solomon/LabelColumns.sql b/ydb/library/yql/tests/sql/suites/solomon/LabelColumns.sql index 0ff5b736c464..217b221a49dd 100644 --- a/ydb/library/yql/tests/sql/suites/solomon/LabelColumns.sql +++ b/ydb/library/yql/tests/sql/suites/solomon/LabelColumns.sql @@ -1,6 +1,6 @@ SELECT * FROM local_solomon.my_project WITH ( program = @@{}@@, - from = "2023-12-08T14:40:39Z", - to = "2023-12-08T14:45:39Z", + from = "1970-01-01T00:00:01Z", + to = "1970-01-02T00:00:01Z", labels = "downsampling.disabled, downsampling.fill, project, downsampling.gridMillis" ); diff --git a/ydb/library/yql/tests/sql/suites/solomon/Subquery.sql b/ydb/library/yql/tests/sql/suites/solomon/Subquery.sql index 40bee34ceec9..2ec636876250 100644 --- a/ydb/library/yql/tests/sql/suites/solomon/Subquery.sql +++ b/ydb/library/yql/tests/sql/suites/solomon/Subquery.sql @@ -2,8 +2,8 @@ DEFINE SUBQUERY $q($p) AS SELECT * FROM local_solomon.my_project WITH ( program = $p, - from = "2023-12-08T14:40:39Z", - to = "2023-12-08T14:45:39Z", + from = "1970-01-01T00:00:01Z", + to = "1970-01-02T00:00:01Z", `downsampling.aggregation` = "SUM", `downsampling.grid_interval` = "25", `downsampling.fill` = "PREVIOUS" diff --git a/ydb/library/yql/tests/sql/suites/solomon/UnknownSetting.sql b/ydb/library/yql/tests/sql/suites/solomon/UnknownSetting.sql index 0e51464d2358..cccd2bca575b 100644 --- a/ydb/library/yql/tests/sql/suites/solomon/UnknownSetting.sql +++ b/ydb/library/yql/tests/sql/suites/solomon/UnknownSetting.sql @@ -1,6 +1,6 @@ SELECT * FROM local_solomon.my_project WITH ( program = @@{}@@, - from = "2023-12-08T14:40:39Z", - to = "2023-12-08T14:45:39Z", + from = "1970-01-01T00:00:01Z", + to = "1970-01-02T00:00:01Z", unk = "ABC" ); diff --git a/ydb/library/yql/tools/solomon_emulator_grpc/__main__.py b/ydb/library/yql/tools/solomon_emulator_grpc/__main__.py index 57479a7b2f11..d7964c554c6f 100644 --- a/ydb/library/yql/tools/solomon_emulator_grpc/__main__.py +++ b/ydb/library/yql/tools/solomon_emulator_grpc/__main__.py @@ -58,7 +58,7 @@ def Read(self, request: ReadRequest, context) -> ReadResponse: timeseries.labels[key] = str(value) timeseries.type = MetricType.RATE - timeseries.timestamp_values.values.extend([1, 2, 3]) + timeseries.timestamp_values.values.extend([10000, 20000, 30000]) timeseries.double_values.values.extend([100, 200, 300]) return response diff --git a/ydb/tests/fq/solomon/canondata/test.test_solomon-Basic-default.txt_/results.txt b/ydb/tests/fq/solomon/canondata/test.test_solomon-Basic-default.txt_/results.txt index 51d34a44bff8..686ff1f6164a 100644 --- a/ydb/tests/fq/solomon/canondata/test.test_solomon-Basic-default.txt_/results.txt +++ b/ydb/tests/fq/solomon/canondata/test.test_solomon-Basic-default.txt_/results.txt @@ -64,7 +64,7 @@ rows { bytes_value: "from" } payload { - bytes_value: "seconds: 1702046439\n" + bytes_value: "seconds: 1\n" } } pairs { @@ -96,12 +96,12 @@ rows { bytes_value: "to" } payload { - bytes_value: "seconds: 1702046739\n" + bytes_value: "seconds: 86401\n" } } } items { - uint32_value: 0 + uint32_value: 10 } items { bytes_value: "RATE" @@ -141,7 +141,7 @@ rows { bytes_value: "from" } payload { - bytes_value: "seconds: 1702046439\n" + bytes_value: "seconds: 1\n" } } pairs { @@ -173,12 +173,12 @@ rows { bytes_value: "to" } payload { - bytes_value: "seconds: 1702046739\n" + bytes_value: "seconds: 86401\n" } } } items { - uint32_value: 0 + uint32_value: 20 } items { bytes_value: "RATE" @@ -218,7 +218,7 @@ rows { bytes_value: "from" } payload { - bytes_value: "seconds: 1702046439\n" + bytes_value: "seconds: 1\n" } } pairs { @@ -250,12 +250,12 @@ rows { bytes_value: "to" } payload { - bytes_value: "seconds: 1702046739\n" + bytes_value: "seconds: 86401\n" } } } items { - uint32_value: 0 + uint32_value: 30 } items { bytes_value: "RATE" diff --git a/ydb/tests/fq/solomon/canondata/test.test_solomon-Downsampling-default.txt_/results.txt b/ydb/tests/fq/solomon/canondata/test.test_solomon-Downsampling-default.txt_/results.txt index d74508502b62..1b03ae9a9d4b 100644 --- a/ydb/tests/fq/solomon/canondata/test.test_solomon-Downsampling-default.txt_/results.txt +++ b/ydb/tests/fq/solomon/canondata/test.test_solomon-Downsampling-default.txt_/results.txt @@ -64,7 +64,7 @@ rows { bytes_value: "from" } payload { - bytes_value: "seconds: 1702046439\n" + bytes_value: "seconds: 1\n" } } pairs { @@ -96,12 +96,12 @@ rows { bytes_value: "to" } payload { - bytes_value: "seconds: 1702046739\n" + bytes_value: "seconds: 86401\n" } } } items { - uint32_value: 0 + uint32_value: 10 } items { bytes_value: "RATE" @@ -141,7 +141,7 @@ rows { bytes_value: "from" } payload { - bytes_value: "seconds: 1702046439\n" + bytes_value: "seconds: 1\n" } } pairs { @@ -173,12 +173,12 @@ rows { bytes_value: "to" } payload { - bytes_value: "seconds: 1702046739\n" + bytes_value: "seconds: 86401\n" } } } items { - uint32_value: 0 + uint32_value: 20 } items { bytes_value: "RATE" @@ -218,7 +218,7 @@ rows { bytes_value: "from" } payload { - bytes_value: "seconds: 1702046439\n" + bytes_value: "seconds: 1\n" } } pairs { @@ -250,12 +250,12 @@ rows { bytes_value: "to" } payload { - bytes_value: "seconds: 1702046739\n" + bytes_value: "seconds: 86401\n" } } } items { - uint32_value: 0 + uint32_value: 30 } items { bytes_value: "RATE" diff --git a/ydb/tests/fq/solomon/canondata/test.test_solomon-DownsamplingValidSettings-default.txt_/results.txt b/ydb/tests/fq/solomon/canondata/test.test_solomon-DownsamplingValidSettings-default.txt_/results.txt index 4a16dad9dab9..3bdc4391abe0 100644 --- a/ydb/tests/fq/solomon/canondata/test.test_solomon-DownsamplingValidSettings-default.txt_/results.txt +++ b/ydb/tests/fq/solomon/canondata/test.test_solomon-DownsamplingValidSettings-default.txt_/results.txt @@ -64,7 +64,7 @@ rows { bytes_value: "from" } payload { - bytes_value: "seconds: 1702046439\n" + bytes_value: "seconds: 1\n" } } pairs { @@ -96,12 +96,12 @@ rows { bytes_value: "to" } payload { - bytes_value: "seconds: 1702046739\n" + bytes_value: "seconds: 86401\n" } } } items { - uint32_value: 0 + uint32_value: 10 } items { bytes_value: "RATE" @@ -141,7 +141,7 @@ rows { bytes_value: "from" } payload { - bytes_value: "seconds: 1702046439\n" + bytes_value: "seconds: 1\n" } } pairs { @@ -173,12 +173,12 @@ rows { bytes_value: "to" } payload { - bytes_value: "seconds: 1702046739\n" + bytes_value: "seconds: 86401\n" } } } items { - uint32_value: 0 + uint32_value: 20 } items { bytes_value: "RATE" @@ -218,7 +218,7 @@ rows { bytes_value: "from" } payload { - bytes_value: "seconds: 1702046439\n" + bytes_value: "seconds: 1\n" } } pairs { @@ -250,12 +250,12 @@ rows { bytes_value: "to" } payload { - bytes_value: "seconds: 1702046739\n" + bytes_value: "seconds: 86401\n" } } } items { - uint32_value: 0 + uint32_value: 30 } items { bytes_value: "RATE" diff --git a/ydb/tests/fq/solomon/canondata/test.test_solomon-HistResponse-default.txt_/results.txt b/ydb/tests/fq/solomon/canondata/test.test_solomon-HistResponse-default.txt_/results.txt index e48e901a8e00..650123b81d6e 100644 --- a/ydb/tests/fq/solomon/canondata/test.test_solomon-HistResponse-default.txt_/results.txt +++ b/ydb/tests/fq/solomon/canondata/test.test_solomon-HistResponse-default.txt_/results.txt @@ -64,7 +64,7 @@ rows { bytes_value: "from" } payload { - bytes_value: "seconds: 1702046439\n" + bytes_value: "seconds: 1\n" } } pairs { @@ -96,12 +96,12 @@ rows { bytes_value: "to" } payload { - bytes_value: "seconds: 1702046739\n" + bytes_value: "seconds: 86401\n" } } } items { - uint32_value: 0 + uint32_value: 10 } items { bytes_value: "RATE" @@ -141,7 +141,7 @@ rows { bytes_value: "from" } payload { - bytes_value: "seconds: 1702046439\n" + bytes_value: "seconds: 1\n" } } pairs { @@ -173,12 +173,12 @@ rows { bytes_value: "to" } payload { - bytes_value: "seconds: 1702046739\n" + bytes_value: "seconds: 86401\n" } } } items { - uint32_value: 0 + uint32_value: 20 } items { bytes_value: "RATE" @@ -218,7 +218,7 @@ rows { bytes_value: "from" } payload { - bytes_value: "seconds: 1702046439\n" + bytes_value: "seconds: 1\n" } } pairs { @@ -250,12 +250,12 @@ rows { bytes_value: "to" } payload { - bytes_value: "seconds: 1702046739\n" + bytes_value: "seconds: 86401\n" } } } items { - uint32_value: 0 + uint32_value: 30 } items { bytes_value: "RATE" diff --git a/ydb/tests/fq/solomon/canondata/test.test_solomon-LabelColumns-default.txt_/results.txt b/ydb/tests/fq/solomon/canondata/test.test_solomon-LabelColumns-default.txt_/results.txt index 2f0927aa4aef..e05f6371c59d 100644 --- a/ydb/tests/fq/solomon/canondata/test.test_solomon-LabelColumns-default.txt_/results.txt +++ b/ydb/tests/fq/solomon/canondata/test.test_solomon-LabelColumns-default.txt_/results.txt @@ -58,7 +58,7 @@ rows { bytes_value: "my_project" } items { - uint32_value: 0 + uint32_value: 10 } items { bytes_value: "RATE" @@ -81,7 +81,7 @@ rows { bytes_value: "my_project" } items { - uint32_value: 0 + uint32_value: 20 } items { bytes_value: "RATE" @@ -104,7 +104,7 @@ rows { bytes_value: "my_project" } items { - uint32_value: 0 + uint32_value: 30 } items { bytes_value: "RATE" From 2c3df8d4029b6bd75b2046c1238b1c34930d141b Mon Sep 17 00:00:00 2001 From: Ivan Sukhov Date: Thu, 31 Jul 2025 15:02:07 +0300 Subject: [PATCH 4/5] Solomon data source usability fixes (#21950) Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../yql/providers/solomon/common/ut/ya.make | 7 ++ .../yql/providers/solomon/common/util.cpp | 114 +++++++++++++++--- .../yql/providers/solomon/common/util.h | 5 +- .../yql/providers/solomon/common/util_ut.cpp | 93 ++++++++++++++ .../yql/providers/solomon/common/ya.make | 5 + .../solomon/proto/dq_solomon_shard.proto | 1 + .../provider/yql_solomon_datasource.cpp | 2 +- .../yql_solomon_datasource_type_ann.cpp | 23 +++- .../provider/yql_solomon_dq_integration.cpp | 12 +- .../provider/yql_solomon_io_discovery.cpp | 24 +++- .../provider/yql_solomon_load_meta.cpp | 10 +- 11 files changed, 251 insertions(+), 45 deletions(-) create mode 100644 ydb/library/yql/providers/solomon/common/ut/ya.make create mode 100644 ydb/library/yql/providers/solomon/common/util_ut.cpp diff --git a/ydb/library/yql/providers/solomon/common/ut/ya.make b/ydb/library/yql/providers/solomon/common/ut/ya.make new file mode 100644 index 000000000000..3865d744f58c --- /dev/null +++ b/ydb/library/yql/providers/solomon/common/ut/ya.make @@ -0,0 +1,7 @@ +UNITTEST_FOR(ydb/library/yql/providers/solomon/common) + +SRCS( + util_ut.cpp +) + +END() diff --git a/ydb/library/yql/providers/solomon/common/util.cpp b/ydb/library/yql/providers/solomon/common/util.cpp index a620b1463097..32136fad9d85 100644 --- a/ydb/library/yql/providers/solomon/common/util.cpp +++ b/ydb/library/yql/providers/solomon/common/util.cpp @@ -6,37 +6,110 @@ #include #include -namespace NYql::NSo { +#include -NSo::NProto::ESolomonClusterType MapClusterType(TSolomonClusterConfig::ESolomonClusterType clusterType) { - switch (clusterType) { - case TSolomonClusterConfig::SCT_SOLOMON: - return NSo::NProto::ESolomonClusterType::CT_SOLOMON; - case TSolomonClusterConfig::SCT_MONITORING: - return NSo::NProto::ESolomonClusterType::CT_MONITORING; - default: - YQL_ENSURE(false, "Invalid cluster type " << ToString(clusterType)); +namespace { + +THolder CompileRE2WithCheck(const std::string& pattern) { + THolder re(new re2::RE2(pattern)); + YQL_ENSURE(re->ok(), "Unable to compile regex " << pattern << ": " << re->error()); + return re; +} + +const TString LABEL_NAME_PATTERN = R"( *[a-zA-Z0-9-._/]{1,50} *)"; +const TString LABEL_VALUE_PATTERN = R"( *"[ -!#-&(-)+->@-_a-{}-~]{1,200}" *)"; + +const TString SENSOR_NAME_PATTERN = "(" + LABEL_VALUE_PATTERN + ")?({.*})"; +THolder SENSOR_NAME_RE = CompileRE2WithCheck(SENSOR_NAME_PATTERN); + +const TString SELECTOR_PATTERN = "(" + LABEL_NAME_PATTERN + "=" + LABEL_VALUE_PATTERN + ")"; +THolder SELECTOR_RE = CompileRE2WithCheck(SELECTOR_PATTERN); + +const TString SELECTORS_FULL_PATTERN = "{((" + SELECTOR_PATTERN + ",)*" + SELECTOR_PATTERN + ")?}"; +THolder SELECTORS_FULL_RE = CompileRE2WithCheck(SELECTORS_FULL_PATTERN); + +TMaybe InsertOrCheck(std::map& selectors, const TString& name, const TString& value) { + auto [it, inserted] = selectors.emplace(name, value); + if (!inserted && it->second != value) { + return TStringBuilder() << "You shouldn't specify \"" << name << "\" label in selectors"; } + return {}; } -std::map ExtractSelectorValues(const TString& selectors) { - YQL_ENSURE(selectors.size() >= 2, "Selectors should be at least 2 characters long"); - std::map result; +} // namespace - auto selectorValues = StringSplitter(selectors.substr(1, selectors.size() - 2)).Split(',').SkipEmpty().ToList(); - for (const auto& selectorValue : selectorValues) { - size_t eqPos = selectorValue.find("="); - YQL_ENSURE(eqPos <= selectorValue.size()); +namespace NYql::NSo { + +TMaybe ParseSelectorValues(const TString& selectors, std::map& result) { + std::optional sensorName; + TString fullBrackets; + if (!RE2::FullMatch(selectors, *SENSOR_NAME_RE, &sensorName, &fullBrackets)) { + return "Selectors should be specified in [\"sensor_name\"]{[label_name1 = \"label_value1\", ...]} format"; + }; + if (sensorName) { + TString name = StripString(*sensorName); + result["name"] = name.substr(1, name.size() - 2); + } + + if (!RE2::FullMatch(fullBrackets, *SELECTORS_FULL_RE)) { + return "Selectors should be specified in [\"sensor_name\"]{[label_name1 = \"label_value1\", ...]} format"; + } + + std::string_view fullBracketsView = fullBrackets; + TString selectorValue; + while (RE2::FindAndConsume(&fullBracketsView, *SELECTOR_RE, &selectorValue)) { + size_t eqPos = selectorValue.find("="); + TString key = StripString(selectorValue.substr(0, eqPos)); TString value = StripString(selectorValue.substr(eqPos + 1, selectorValue.size() - eqPos - 1)); - YQL_ENSURE(!key.empty()); - YQL_ENSURE(value.size() >= 2); - + result[key] = value.substr(1, value.size() - 2); } - return result; + return {}; +} + +TMaybe BuildSelectorValues(const NSo::NProto::TDqSolomonSource& source, const TString& selectors, std::map& result) { + #define RET_ON_ERROR(expr) \ + if (auto error = expr) { \ + return error; \ + } + + RET_ON_ERROR(ParseSelectorValues(selectors, result)); + + if (source.GetClusterType() == NSo::NProto::CT_MONITORING) { + RET_ON_ERROR(InsertOrCheck(result, "cloudId", source.GetProject())); + RET_ON_ERROR(InsertOrCheck(result, "folderId", source.GetCluster())); + RET_ON_ERROR(InsertOrCheck(result, "service", source.GetService())); + } else { + RET_ON_ERROR(InsertOrCheck(result, "project", source.GetProject())); + } + + if (source.GetClusterType() == NSo::NProto::CT_MONITORING) { + if (auto it = result.find("cloudId"); it != result.end()) { + result["project"] = it->second; + result.erase(it); + } + if (auto it = result.find("folderId"); it != result.end()) { + result["cluster"] = it->second; + result.erase(it); + } + } + + #undef RET_ON_ERROR + return {}; +} + +NSo::NProto::ESolomonClusterType MapClusterType(TSolomonClusterConfig::ESolomonClusterType clusterType) { + switch (clusterType) { + case TSolomonClusterConfig::SCT_SOLOMON: + return NSo::NProto::ESolomonClusterType::CT_SOLOMON; + case TSolomonClusterConfig::SCT_MONITORING: + return NSo::NProto::ESolomonClusterType::CT_MONITORING; + default: + YQL_ENSURE(false, "Invalid cluster type " << ToString(clusterType)); + } } NProto::TDqSolomonSource FillSolomonSource(const TSolomonClusterConfig* config, const TString& project) { @@ -48,6 +121,7 @@ NProto::TDqSolomonSource FillSolomonSource(const TSolomonClusterConfig* config, if (source.GetClusterType() == NSo::NProto::CT_MONITORING) { source.SetProject(config->GetPath().GetProject()); source.SetCluster(config->GetPath().GetCluster()); + source.SetService(project); } else { source.SetProject(project); } diff --git a/ydb/library/yql/providers/solomon/common/util.h b/ydb/library/yql/providers/solomon/common/util.h index b88435f42360..8b2e8ac72b7b 100644 --- a/ydb/library/yql/providers/solomon/common/util.h +++ b/ydb/library/yql/providers/solomon/common/util.h @@ -5,10 +5,11 @@ namespace NYql::NSo { +TMaybe ParseSelectorValues(const TString& selectors, std::map& result); +TMaybe BuildSelectorValues(const NSo::NProto::TDqSolomonSource& source, const TString& selectors, std::map& result); + NSo::NProto::ESolomonClusterType MapClusterType(TSolomonClusterConfig::ESolomonClusterType clusterType); -std::map ExtractSelectorValues(const TString& selectors); - NProto::TDqSolomonSource FillSolomonSource(const TSolomonClusterConfig* config, const TString& project); } // namespace NYql::NSo diff --git a/ydb/library/yql/providers/solomon/common/util_ut.cpp b/ydb/library/yql/providers/solomon/common/util_ut.cpp new file mode 100644 index 000000000000..ccd65ede0b7d --- /dev/null +++ b/ydb/library/yql/providers/solomon/common/util_ut.cpp @@ -0,0 +1,93 @@ +#include "util.h" + +#include + +namespace NYql::NSo { + +Y_UNIT_TEST_SUITE(TestSolomonParseSelectors) { + Y_UNIT_TEST(Basic) { + TString selectors = "{a = \"a\", b = \"b\"}"; + std::map result; + + std::map expectedResult = { + { "b", "b" }, + { "a", "a" } + }; + + UNIT_ASSERT_EQUAL(ParseSelectorValues(selectors, result), TMaybe{}); + UNIT_ASSERT_EQUAL(result, expectedResult); + } + + Y_UNIT_TEST(NewFormat) { + TString selectors = "\"sensor_name\"{a = \"a\", b = \"b\"}"; + std::map result; + + std::map expectedResult = { + { "a", "a" }, + { "b", "b" }, + { "name", "sensor_name" } + }; + + UNIT_ASSERT_EQUAL(ParseSelectorValues(selectors, result), TMaybe{}); + UNIT_ASSERT_EQUAL(result, expectedResult); + } + + Y_UNIT_TEST(Empty) { + TString selectors = "{}"; + std::map result; + + std::map expectedResult = {}; + + UNIT_ASSERT_EQUAL(ParseSelectorValues(selectors, result), TMaybe{}); + UNIT_ASSERT_EQUAL(result, expectedResult); + } + + Y_UNIT_TEST(EmptyNewFormat) { + TString selectors = "\"sensor_name\"{}"; + std::map result; + + std::map expectedResult = { + { "name", "sensor_name" } + }; + + UNIT_ASSERT_EQUAL(ParseSelectorValues(selectors, result), TMaybe{}); + UNIT_ASSERT_EQUAL(result, expectedResult); + } + + Y_UNIT_TEST(NoBrackets) { + TString selectors = "a = \"a\", b = \"b\""; + std::map result; + + UNIT_ASSERT_EQUAL(ParseSelectorValues(selectors, result), "Selectors should be specified in [\"sensor_name\"]{[label_name1 = \"label_value1\", ...]} format"); + } + + Y_UNIT_TEST(NoQuotes) { + TString selectors = "{a = a, b = b}"; + std::map result; + + UNIT_ASSERT_EQUAL(ParseSelectorValues(selectors, result), "Selectors should be specified in [\"sensor_name\"]{[label_name1 = \"label_value1\", ...]} format"); + } + + Y_UNIT_TEST(NoQuotesOnSensorName) { + TString selectors = "sensor_name{a = \"a\", b = \"b\"}"; + std::map result; + + UNIT_ASSERT_EQUAL(ParseSelectorValues(selectors, result), "Selectors should be specified in [\"sensor_name\"]{[label_name1 = \"label_value1\", ...]} format"); + } + + Y_UNIT_TEST(ValidLabelValues) { + TString selectors = "\"{\"{a = \",\", b = \"}\"}"; + std::map result; + + std::map expectedResult = { + { "name", "{" }, + { "a", "," }, + { "b", "}" } + }; + + UNIT_ASSERT_EQUAL(ParseSelectorValues(selectors, result), TMaybe{}); + UNIT_ASSERT_EQUAL(result, expectedResult); + } +} + +} // namespace NYql::NSo diff --git a/ydb/library/yql/providers/solomon/common/ya.make b/ydb/library/yql/providers/solomon/common/ya.make index 41c6f18964dd..b93c21e592f3 100644 --- a/ydb/library/yql/providers/solomon/common/ya.make +++ b/ydb/library/yql/providers/solomon/common/ya.make @@ -5,9 +5,14 @@ SRCS( ) PEERDIR( + contrib/libs/re2 ydb/library/yql/providers/solomon/proto yql/essentials/providers/common/proto yql/essentials/utils ) END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/library/yql/providers/solomon/proto/dq_solomon_shard.proto b/ydb/library/yql/providers/solomon/proto/dq_solomon_shard.proto index cab88ac20c0d..75a20b293e26 100644 --- a/ydb/library/yql/providers/solomon/proto/dq_solomon_shard.proto +++ b/ydb/library/yql/providers/solomon/proto/dq_solomon_shard.proto @@ -68,4 +68,5 @@ message TDqSolomonSource { string HttpEndpoint = 16; string GrpcEndpoint = 17; optional string Cluster = 18; + optional string Service = 19; } diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource.cpp index 7cb1775415e0..52871bb18745 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource.cpp @@ -92,7 +92,7 @@ class TSolomonDataSource : public TDataProviderBase { if (node.IsCallable(TCoDataSource::CallableName())) { if (node.Child(0)->Content() == SolomonProviderName) { auto clusterName = node.Child(1)->Content(); - if (!State_->Gateway->HasCluster(clusterName)) { + if (clusterName != NCommon::ALL_CLUSTERS && !State_->Gateway->HasCluster(clusterName)) { ctx.AddError(TIssue(ctx.GetPosition(node.Child(1)->Pos()), TStringBuilder() << "Unknown cluster name: " << clusterName)); return false; diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_type_ann.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_type_ann.cpp index 6cf6e144e7ab..415ae32461d0 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_type_ann.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_type_ann.cpp @@ -34,6 +34,7 @@ class TSolomonDataSourceTypeAnnotationTransformer : public TVisitorTransformerBa AddHandler({TSoReadObject::CallableName()}, Hndl(&TSelf::HandleRead)); AddHandler({TSoObject::CallableName()}, Hndl(&TSelf::HandleSoObject)); AddHandler({TSoSourceSettings::CallableName()}, Hndl(&TSelf::HandleSoSourceSettings)); + AddHandler({TCoConfigure::CallableName()}, Hndl(&TSelf::HandleConfig)); } TStatus HandleSoSourceSettings(const TExprNode::TPtr& input, TExprContext& ctx) { @@ -107,11 +108,6 @@ class TSolomonDataSourceTypeAnnotationTransformer : public TVisitorTransformerBa return TStatus::Error; } - if (!hasSelectors && !hasProgram) { - ctx.AddError(TIssue(ctx.GetPosition(selectors.Pos()), "specify either program or selectors")); - return TStatus::Error; - } - auto& downsamplingDisabled = *input->Child(TSoSourceSettings::idx_DownsamplingDisabled); if (!downsamplingDisabled.IsCallable("Bool")) { ctx.AddError(TIssue(ctx.GetPosition(downsamplingDisabled.Pos()), "downsampling.disabled must be bool")); @@ -226,6 +222,23 @@ class TSolomonDataSourceTypeAnnotationTransformer : public TVisitorTransformerBa return TStatus::Ok; } + TStatus HandleConfig(const TExprNode::TPtr& input, TExprContext& ctx) { + if (!EnsureMinArgsCount(*input, 2, ctx)) { + return TStatus::Error; + } + + if (!EnsureWorldType(*input->Child(TCoConfigure::idx_World), ctx)) { + return TStatus::Error; + } + + if (!EnsureSpecificDataSource(*input->Child(TCoConfigure::idx_DataSource), SolomonProviderName, ctx)) { + return TStatus::Error; + } + + input->SetTypeAnn(input->Child(TCoConfigure::idx_World)->GetTypeAnn()); + return TStatus::Ok; + } + private: TSolomonState::TPtr State_; }; diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp index a71e49b7e364..7a688f285e2b 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp @@ -292,15 +292,11 @@ class TSolomonDqIntegration: public TDqIntegrationBase { auto selectors = settings.Selectors().StringValue(); if (!selectors.empty()) { - auto labelValues = NSo::ExtractSelectorValues(selectors); - if (source.GetClusterType() == NSo::NProto::CT_MONITORING) { - labelValues.insert({ "service", settings.Project().StringValue() }); - labelValues.insert({ "cluster", source.GetCluster() }); - } else { - labelValues.insert({ "project", source.GetProject() }); + std::map selectorValues; + if (auto error = NSo::BuildSelectorValues(source, selectors, selectorValues)) { + throw yexception() << *error; } - - source.MutableSelectors()->insert(labelValues.begin(), labelValues.end()); + source.MutableSelectors()->insert(selectorValues.begin(), selectorValues.end()); } auto program = settings.Program().StringValue(); diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_io_discovery.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_io_discovery.cpp index 230bcc706af6..10845f6114d2 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_io_discovery.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_io_discovery.cpp @@ -41,6 +41,16 @@ TVector ExtractUserLabels(TPositionHandle pos, TExprContext& ctx, TExpr return {}; } +bool HasSetting(TExprNode::TListType& settings, const TString& name) { + for (auto it = settings.cbegin(); settings.cend() != it; ++it) { + if (const auto item = *it; item->Head().IsAtom(name)) { + return true; + } + } + + return false; +} + const TStructExprType* BuildScheme(TPositionHandle pos, const TVector& userLabels, TExprContext& ctx, TVector& systemColumns) { auto allSystemColumns = {SOLOMON_SCHEME_LABELS, SOLOMON_SCHEME_VALUE, SOLOMON_SCHEME_TS, SOLOMON_SCHEME_TYPE}; TVector columnTypes; @@ -135,9 +145,17 @@ class TSolomonIODiscoveryTransformer : public TSyncTransformerBase { auto userSchema = ExtractSchema(settingsList); TVector userLabels = ExtractUserLabels(settings->Pos(), ctx, settingsList); - auto newSettings = Build(ctx, settings->Pos()) - .Add(settingsList) - .Done(); + auto newSettingsBuilder = Build(ctx, settings->Pos()) + .Add(settingsList); + + if (!HasSetting(settingsList, "program") && !HasSetting(settingsList, "selectors")) { + newSettingsBuilder.Add() + .Name().Build("selectors") + .Value().Build("{}") + .Build(); + } + + auto newSettings = newSettingsBuilder.Done(); auto soObject = Build(ctx, read.Pos()) .Project().Build(project) diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_load_meta.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_load_meta.cpp index 847e3b2836ad..2ef1b36d8b99 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_load_meta.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_load_meta.cpp @@ -81,12 +81,10 @@ class TSolomonLoadTableMetadataTransformer : public TGraphTransformerBase { to = TInstant::Now(); } - auto selectors = NSo::ExtractSelectorValues(*maybeSelectors); - if (source.GetClusterType() == NSo::NProto::CT_MONITORING) { - selectors["cluster"] = source.GetCluster(); - selectors["service"] = soReadObject.Object().Project().StringValue(); - } else { - selectors["project"] = source.GetProject(); + std::map selectors; + if (auto error = NSo::BuildSelectorValues(source, *maybeSelectors, selectors)) { + ctx.AddError(TIssue(ctx.GetPosition(n->Pos()), *error)); + return TStatus::Error; } auto defaultReplica = (source.GetClusterType() == NSo::NProto::CT_SOLOMON ? "sas" : "cloud-prod-a"); From 27b34ace3497f2c10f786783eca59eed93a09a11 Mon Sep 17 00:00:00 2001 From: Ivan Sukhov Date: Wed, 6 Aug 2025 02:21:52 +0300 Subject: [PATCH 5/5] solomon: working inflight restriction, retry policy tweaks & quality of life changes (#22358) --- .../solomon/actors/dq_solomon_read_actor.cpp | 47 ++++++++-- .../yql/providers/solomon/common/util.cpp | 25 ++++- .../yql/providers/solomon/common/util.h | 2 + .../yql/providers/solomon/common/util_ut.cpp | 94 +++++++++++++++++++ .../expr_nodes/yql_solomon_expr_nodes.json | 12 ++- .../solomon/proto/dq_solomon_shard.proto | 1 + .../yql_solomon_datasource_type_ann.cpp | 14 ++- .../provider/yql_solomon_dq_integration.cpp | 8 +- .../provider/yql_solomon_io_discovery.cpp | 44 +++++++-- .../client/solomon_accessor_client.cpp | 7 +- .../opt.yql | 2 +- .../opt.yql | 2 +- .../opt.yql | 2 +- .../opt.yql | 2 +- .../opt.yql | 2 +- .../opt.yql | 2 +- .../opt.yql | 2 +- 17 files changed, 227 insertions(+), 41 deletions(-) diff --git a/ydb/library/yql/providers/solomon/actors/dq_solomon_read_actor.cpp b/ydb/library/yql/providers/solomon/actors/dq_solomon_read_actor.cpp index b60b298860ff..2ed50a6cff6b 100644 --- a/ydb/library/yql/providers/solomon/actors/dq_solomon_read_actor.cpp +++ b/ydb/library/yql/providers/solomon/actors/dq_solomon_read_actor.cpp @@ -82,6 +82,7 @@ class TDqSolomonReadActor : public NActors::TActorBootstrapped credentialsProvider @@ -97,6 +98,7 @@ class TDqSolomonReadActor : public NActors::TActorBootstrapped names(ReadParams.Source.GetSystemColumns().begin(), ReadParams.Source.GetSystemColumns().end()); - names.insert(names.end(), ReadParams.Source.GetLabelNames().begin(), ReadParams.Source.GetLabelNames().end()); + names.insert(names.end(), ReadParams.Source.GetLabelNameAliases().begin(), ReadParams.Source.GetLabelNameAliases().end()); std::sort(names.begin(), names.end()); size_t index = 0; for (auto& n : names) { @@ -244,7 +252,7 @@ class TDqSolomonReadActor : public NActors::TActorBootstrappedsecond] = dictValue; } - for (const auto& c : ReadParams.Source.GetLabelNames()) { + for (const auto& c : ReadParams.Source.GetLabelNameAliases()) { auto& v = items[Index[c]]; - auto it = labels.find(c); + auto it = labels.find(AliasIndex[c]); if (it != labels.end()) { v = NKikimr::NMiniKQL::MakeString(it->second); } else { @@ -432,6 +440,7 @@ class TDqSolomonReadActor : public NActors::TActorBootstrapped= MaxInflight) { + return false; + } + + RequestData(); + return true; } void RequestData() { @@ -471,6 +487,7 @@ class TDqSolomonReadActor : public NActors::TActorBootstrappedGetData(request.Selectors, request.From, request.To); @@ -535,6 +552,7 @@ class TDqSolomonReadActor : public NActors::TActorBootstrapped::TPtr RetryPolicy; bool UseMetricsQueue; @@ -583,6 +602,7 @@ class TDqSolomonReadActor : public NActors::TActorBootstrapped SystemColumnPositionIndex; THashMap Index; + THashMap AliasIndex; }; @@ -636,6 +657,11 @@ std::pair CreateDqSolom truePointsFindRange = FromString(it->second); } + ui64 maxInflight = 40; + if (auto it = settings.find("maxApiInflight"); it != settings.end()) { + maxInflight = FromString(it->second); + } + auto credentialsProviderFactory = CreateCredentialsProviderFactoryForStructuredToken(credentialsFactory, token); auto credentialsProvider = credentialsProviderFactory->CreateProvider(); @@ -650,6 +676,7 @@ std::pair CreateDqSolom computeActorBatchSize, TDuration::Seconds(truePointsFindRange), metricsQueueConsumersCountDelta, + maxInflight, metricsQueueActor, counters, credentialsProvider); diff --git a/ydb/library/yql/providers/solomon/common/util.cpp b/ydb/library/yql/providers/solomon/common/util.cpp index 32136fad9d85..5a8c19445664 100644 --- a/ydb/library/yql/providers/solomon/common/util.cpp +++ b/ydb/library/yql/providers/solomon/common/util.cpp @@ -17,7 +17,7 @@ THolder CompileRE2WithCheck(const std::string& pattern) { } const TString LABEL_NAME_PATTERN = R"( *[a-zA-Z0-9-._/]{1,50} *)"; -const TString LABEL_VALUE_PATTERN = R"( *"[ -!#-&(-)+->@-_a-{}-~]{1,200}" *)"; +const TString LABEL_VALUE_PATTERN = R"( *"[ -!#-&(-)+->@-_a-{}-~*|-]{1,200}" *)"; const TString SENSOR_NAME_PATTERN = "(" + LABEL_VALUE_PATTERN + ")?({.*})"; THolder SENSOR_NAME_RE = CompileRE2WithCheck(SENSOR_NAME_PATTERN); @@ -28,6 +28,9 @@ THolder SELECTOR_RE = CompileRE2WithCheck(SELECTOR_PATTERN); const TString SELECTORS_FULL_PATTERN = "{((" + SELECTOR_PATTERN + ",)*" + SELECTOR_PATTERN + ")?}"; THolder SELECTORS_FULL_RE = CompileRE2WithCheck(SELECTORS_FULL_PATTERN); +const TString USER_LABELS_PATTERN = "(" + LABEL_NAME_PATTERN + ")(?: (?i:as) (" + LABEL_NAME_PATTERN + "))?"; +THolder USER_LABELS_RE = CompileRE2WithCheck(USER_LABELS_PATTERN); + TMaybe InsertOrCheck(std::map& selectors, const TString& name, const TString& value) { auto [it, inserted] = selectors.emplace(name, value); if (!inserted && it->second != value) { @@ -101,6 +104,26 @@ TMaybe BuildSelectorValues(const NSo::NProto::TDqSolomonSource& source, return {}; } +TMaybe ParseLabelNames(const TString& labelNames, TVector& names, TVector& aliases) { + auto labels = StringSplitter(labelNames).Split(',').SkipEmpty().ToList(); + names.reserve(labels.size()); + aliases.reserve(labels.size()); + + for (TString& label : labels) { + TString name; + std::optional alias; + + if (!RE2::FullMatch(label, *USER_LABELS_RE, &name, &alias)) { + return "Label names should be specified in \"label1 [as alias1], label2 [as alias2], ...\" format"; + } + + names.push_back(StripString(name)); + aliases.push_back(StripString(alias ? *alias : name)); + } + + return {}; +} + NSo::NProto::ESolomonClusterType MapClusterType(TSolomonClusterConfig::ESolomonClusterType clusterType) { switch (clusterType) { case TSolomonClusterConfig::SCT_SOLOMON: diff --git a/ydb/library/yql/providers/solomon/common/util.h b/ydb/library/yql/providers/solomon/common/util.h index 8b2e8ac72b7b..449c1dd4ec1d 100644 --- a/ydb/library/yql/providers/solomon/common/util.h +++ b/ydb/library/yql/providers/solomon/common/util.h @@ -7,6 +7,8 @@ namespace NYql::NSo { TMaybe ParseSelectorValues(const TString& selectors, std::map& result); TMaybe BuildSelectorValues(const NSo::NProto::TDqSolomonSource& source, const TString& selectors, std::map& result); + +TMaybe ParseLabelNames(const TString& labelNames, TVector& names, TVector& aliases); NSo::NProto::ESolomonClusterType MapClusterType(TSolomonClusterConfig::ESolomonClusterType clusterType); diff --git a/ydb/library/yql/providers/solomon/common/util_ut.cpp b/ydb/library/yql/providers/solomon/common/util_ut.cpp index ccd65ede0b7d..5e3320161eb7 100644 --- a/ydb/library/yql/providers/solomon/common/util_ut.cpp +++ b/ydb/library/yql/providers/solomon/common/util_ut.cpp @@ -90,4 +90,98 @@ Y_UNIT_TEST_SUITE(TestSolomonParseSelectors) { } } +Y_UNIT_TEST_SUITE(TestSolomonParseLabelNames) { + Y_UNIT_TEST(Basic) { + TString labelNames = "label1, label2"; + TVector names; + TVector aliases; + + TVector expectedNames = { + "label1", "label2" + }; + TVector expectedAliases = { + "label1", "label2" + }; + + UNIT_ASSERT_EQUAL(ParseLabelNames(labelNames, names, aliases), TMaybe{}); + UNIT_ASSERT_EQUAL(names, expectedNames); + UNIT_ASSERT_EQUAL(aliases, expectedAliases); + } + + Y_UNIT_TEST(WithAliases) { + TString labelNames = "label1 as alias1, label2 as alias2"; + TVector names; + TVector aliases; + + TVector expectedNames = { + "label1", "label2" + }; + TVector expectedAliases = { + "alias1", "alias2" + }; + + UNIT_ASSERT_EQUAL(ParseLabelNames(labelNames, names, aliases), TMaybe{}); + UNIT_ASSERT_EQUAL(names, expectedNames); + UNIT_ASSERT_EQUAL(aliases, expectedAliases); + } + + Y_UNIT_TEST(OneAlias) { + TString labelNames = "label1, label2 as alias2, label3"; + TVector names; + TVector aliases; + + TVector expectedNames = { + "label1", "label2", "label3" + }; + TVector expectedAliases = { + "label1", "alias2", "label3" + }; + + UNIT_ASSERT_EQUAL(ParseLabelNames(labelNames, names, aliases), TMaybe{}); + UNIT_ASSERT_EQUAL(names, expectedNames); + UNIT_ASSERT_EQUAL(aliases, expectedAliases); + } + + Y_UNIT_TEST(CaseSensitivity) { + TString labelNames = "label1, label2 AS alias2, label3"; + TVector names; + TVector aliases; + + TVector expectedNames = { + "label1", "label2", "label3" + }; + TVector expectedAliases = { + "label1", "alias2", "label3" + }; + + UNIT_ASSERT_EQUAL(ParseLabelNames(labelNames, names, aliases), TMaybe{}); + UNIT_ASSERT_EQUAL(names, expectedNames); + UNIT_ASSERT_EQUAL(aliases, expectedAliases); + } + + Y_UNIT_TEST(InvalidLabelName) { + TString labelNames = "{}, {}"; + TVector names; + TVector aliases; + + UNIT_ASSERT_EQUAL(ParseLabelNames(labelNames, names, aliases), "Label names should be specified in \"label1 [as alias1], label2 [as alias2], ...\" format"); + } + + Y_UNIT_TEST(NoAs) { + TString labelNames = "label1 alias1"; + TVector names; + TVector aliases; + + UNIT_ASSERT_EQUAL(ParseLabelNames(labelNames, names, aliases), "Label names should be specified in \"label1 [as alias1], label2 [as alias2], ...\" format"); + } + + Y_UNIT_TEST(EmptyAlias) { + TString labelNames = "label1 as, label2"; + TVector names; + TVector aliases; + + UNIT_ASSERT_EQUAL(ParseLabelNames(labelNames, names, aliases), "Label names should be specified in \"label1 [as alias1], label2 [as alias2], ...\" format"); + } +} + } // namespace NYql::NSo diff --git a/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.json b/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.json index f19b5c256c29..a23eb6c22289 100644 --- a/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.json +++ b/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.json @@ -63,7 +63,8 @@ {"Index": 12, "Name": "DownsamplingFill", "Type": "TCoAtom"}, {"Index": 13, "Name": "DownsamplingGridSec", "Type": "TCoUint32"}, {"Index": 14, "Name": "RequiredLabelNames", "Type": "TCoAtomList"}, - {"Index": 15, "Name": "TotalMetricsCount", "Type": "TCoAtom"} + {"Index": 15, "Name": "TotalMetricsCount", "Type": "TCoAtom"}, + {"Index": 16, "Name": "LabelNameAliases", "Type": "TCoAtomList"} ] }, { @@ -85,10 +86,11 @@ {"Index": 2, "Name": "Object", "Type": "TSoObject"}, {"Index": 3, "Name": "SystemColumns", "Type": "TCoAtomList"}, {"Index": 4, "Name": "LabelNames", "Type": "TCoAtomList"}, - {"Index": 5, "Name": "RequiredLabelNames", "Type": "TCoAtomList"}, - {"Index": 6, "Name": "TotalMetricsCount", "Type": "TCoAtom"}, - {"Index": 7, "Name": "RowType", "Type": "TExprBase"}, - {"Index": 8, "Name": "ColumnOrder", "Type": "TExprBase", "Optional": true} + {"Index": 5, "Name": "LabelNameAliases", "Type": "TCoAtomList"}, + {"Index": 6, "Name": "RequiredLabelNames", "Type": "TCoAtomList"}, + {"Index": 7, "Name": "TotalMetricsCount", "Type": "TCoAtom"}, + {"Index": 8, "Name": "RowType", "Type": "TExprBase"}, + {"Index": 9, "Name": "ColumnOrder", "Type": "TExprBase", "Optional": true} ] }, { diff --git a/ydb/library/yql/providers/solomon/proto/dq_solomon_shard.proto b/ydb/library/yql/providers/solomon/proto/dq_solomon_shard.proto index 75a20b293e26..e90555a416d0 100644 --- a/ydb/library/yql/providers/solomon/proto/dq_solomon_shard.proto +++ b/ydb/library/yql/providers/solomon/proto/dq_solomon_shard.proto @@ -69,4 +69,5 @@ message TDqSolomonSource { string GrpcEndpoint = 17; optional string Cluster = 18; optional string Service = 19; + repeated string LabelNameAliases = 20; } diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_type_ann.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_type_ann.cpp index 415ae32461d0..75b022b02aeb 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_type_ann.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_type_ann.cpp @@ -38,7 +38,7 @@ class TSolomonDataSourceTypeAnnotationTransformer : public TVisitorTransformerBa } TStatus HandleSoSourceSettings(const TExprNode::TPtr& input, TExprContext& ctx) { - if (!EnsureArgsCount(*input, 16, ctx)) { + if (!EnsureArgsCount(*input, 17, ctx)) { return TStatus::Error; } @@ -71,6 +71,11 @@ class TSolomonDataSourceTypeAnnotationTransformer : public TVisitorTransformerBa return TStatus::Error; } + auto& labelNameAliases = *input->Child(TSoSourceSettings::idx_LabelNameAliases); + if (!EnsureTupleOfAtoms(labelNameAliases, ctx)) { + return TStatus::Error; + } + auto& requiredLabelNames = *input->Child(TSoSourceSettings::idx_RequiredLabelNames); if (!EnsureTupleOfAtoms(requiredLabelNames, ctx)) { return TStatus::Error; @@ -155,7 +160,7 @@ class TSolomonDataSourceTypeAnnotationTransformer : public TVisitorTransformerBa } TStatus HandleRead(const TExprNode::TPtr& input, TExprContext& ctx) { - if (!EnsureMinMaxArgsCount(*input, 8U, 9U, ctx)) { + if (!EnsureMinMaxArgsCount(*input, 9U, 10U, ctx)) { return TStatus::Error; } @@ -177,6 +182,11 @@ class TSolomonDataSourceTypeAnnotationTransformer : public TVisitorTransformerBa return TStatus::Error; } + auto& labelNameAliases = *input->Child(TSoReadObject::idx_LabelNameAliases); + if (!EnsureTupleOfAtoms(labelNameAliases, ctx)) { + return TStatus::Error; + } + auto& requiredLabelNames = *input->Child(TSoReadObject::idx_RequiredLabelNames); if (!EnsureTupleOfAtoms(requiredLabelNames, ctx)) { return TStatus::Error; diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp index 7a688f285e2b..5dfae73a2eef 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp @@ -260,6 +260,7 @@ class TSolomonDqIntegration: public TDqIntegrationBase { .DownsamplingFill().Build(downsamplingFill ? *downsamplingFill : "") .DownsamplingGridSec().Literal().Build(ToString(downsamplingGridSec ? *downsamplingGridSec : 0)).Build() .TotalMetricsCount(soReadObject.TotalMetricsCount()) + .LabelNameAliases(soReadObject.LabelNameAliases()) .Build() .DataSource(soReadObject.DataSource().Cast()) .RowType(soReadObject.RowType()) @@ -322,11 +323,16 @@ class TSolomonDqIntegration: public TDqIntegrationBase { } for (const auto& c : settings.LabelNames()) { + const auto& columnAsString = c.StringValue(); + source.AddLabelNames(columnAsString); + } + + for (const auto& c : settings.LabelNameAliases()) { const auto& columnAsString = c.StringValue(); if (!uniqueColumns.insert(columnAsString).second) { throw yexception() << "Column " << columnAsString << " already registered"; } - source.AddLabelNames(columnAsString); + source.AddLabelNameAliases(columnAsString); } for (const auto& c : settings.RequiredLabelNames()) { diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_io_discovery.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_io_discovery.cpp index 10845f6114d2..612c8c3b434b 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_io_discovery.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_io_discovery.cpp @@ -1,6 +1,7 @@ #include "yql_solomon_provider_impl.h" #include +#include #include #include @@ -23,18 +24,30 @@ std::array ExtractSchema(TExprNode::TListType& settings) { return {}; } -TVector ExtractUserLabels(TPositionHandle pos, TExprContext& ctx, TExprNode::TListType& settings) { +TMaybe ExtractUserLabels(TPositionHandle pos, TExprContext& ctx, TExprNode::TListType& settings, TVector& names, TVector& aliases) { + names.clear(); + aliases.clear(); + for (auto it = settings.cbegin(); settings.cend() != it; ++it) { if (const auto item = *it; item->Head().IsAtom("labels")) { - TVector result; - auto labels = StringSplitter(TString(item->Tail().Content())).Split(',').SkipEmpty().ToList(); - result.reserve(labels.size()); - for (TString& label : labels) { - auto v = Build(ctx, pos).Value(StripString(label)).Done(); - result.emplace_back(std::move(v)); + TVector stringNames; + TVector stringAliases; + if (auto error = NSo::ParseLabelNames(TString(item->Tail().Content()), stringNames, stringAliases)) { + return error; } + + names.reserve(stringNames.size()); + aliases.reserve(stringAliases.size()); + for (size_t i = 0; i < stringNames.size(); ++i) { + auto name = Build(ctx, pos).Value(stringNames[i]).Done(); + auto alias = Build(ctx, pos).Value(stringAliases[i]).Done(); + + names.emplace_back(std::move(name)); + aliases.emplace_back(std::move(alias)); + } + settings.erase(it); - return result; + return {}; } } @@ -143,7 +156,12 @@ class TSolomonIODiscoveryTransformer : public TSyncTransformerBase { auto settings = read.Ref().Child(4); auto settingsList = read.Ref().Child(4)->ChildrenList(); auto userSchema = ExtractSchema(settingsList); - TVector userLabels = ExtractUserLabels(settings->Pos(), ctx, settingsList); + TVector userLabels; + TVector userLabelAliases; + if (auto error = ExtractUserLabels(settings->Pos(), ctx, settingsList, userLabels, userLabelAliases)) { + ctx.AddError(TIssue(ctx.GetPosition(settings->Pos()), TStringBuilder() << "Invalid label names: " << *error)); + return {}; + } auto newSettingsBuilder = Build(ctx, settings->Pos()) .Add(settingsList); @@ -163,7 +181,7 @@ class TSolomonIODiscoveryTransformer : public TSyncTransformerBase { .Done(); TVector systemColumns; - auto* scheme = BuildScheme(settings->Pos(), userLabels, ctx, systemColumns); + auto* scheme = BuildScheme(settings->Pos(), userLabelAliases, ctx, systemColumns); if (!scheme) { return {}; } @@ -176,6 +194,10 @@ class TSolomonIODiscoveryTransformer : public TSyncTransformerBase { auto labelNamesNode = Build(ctx, read.Pos()) .Add(userLabels) .Done(); + + auto labelNameAliasesNode = Build(ctx, read.Pos()) + .Add(userLabelAliases) + .Done(); return userSchema.back() ? Build(ctx, read.Pos()) @@ -184,6 +206,7 @@ class TSolomonIODiscoveryTransformer : public TSyncTransformerBase { .Object(soObject) .SystemColumns(systemColumnsNode) .LabelNames(labelNamesNode) + .LabelNameAliases(labelNameAliasesNode) .RequiredLabelNames().Build() .TotalMetricsCount().Build() .RowType(rowTypeNode) @@ -195,6 +218,7 @@ class TSolomonIODiscoveryTransformer : public TSyncTransformerBase { .Object(soObject) .SystemColumns(systemColumnsNode) .LabelNames(labelNamesNode) + .LabelNameAliases(labelNameAliasesNode) .RequiredLabelNames().Build() .TotalMetricsCount().Build() .RowType(rowTypeNode) diff --git a/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.cpp b/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.cpp index af08c07f11f5..9bc2e59ff573 100644 --- a/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.cpp +++ b/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.cpp @@ -243,16 +243,14 @@ class TSolomonAccessorClient : public ISolomonAccessorClient, public std::enable NYql::NSo::NProto::TDqSolomonSource&& settings, std::shared_ptr credentialsProvider) : DefaultReplica(defaultReplica) - , MaxApiInflight(maxApiInflight) , Settings(std::move(settings)) , CredentialsProvider(credentialsProvider) { - HttpConfig.SetMaxInFlightCount(MaxApiInflight); + HttpConfig.SetMaxInFlightCount(maxApiInflight); HttpGateway = IHTTPGateway::Make(&HttpConfig); GrpcConfig.Locator = GetGrpcSolomonEndpoint(); GrpcConfig.EnableSsl = Settings.GetUseSsl(); - GrpcConfig.MaxInFlight = MaxApiInflight; GrpcClient = std::make_shared(); GrpcConnection = GrpcClient->CreateGRpcServiceConnection(GrpcConfig); } @@ -425,7 +423,7 @@ class TSolomonAccessorClient : public ISolomonAccessorClient, public std::enable }, TDuration::MilliSeconds(25), TDuration::MilliSeconds(200), - TDuration::MilliSeconds(500), + TDuration::MilliSeconds(1000), 10 ); @@ -584,7 +582,6 @@ class TSolomonAccessorClient : public ISolomonAccessorClient, public std::enable private: const TString DefaultReplica; const ui64 ListSizeLimit = 1ull << 20; - const ui64 MaxApiInflight; const NYql::NSo::NProto::TDqSolomonSource Settings; const std::shared_ptr CredentialsProvider; diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Basic-default.txt_/opt.yql b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Basic-default.txt_/opt.yql index d79f0660e437..00fb2d36efa5 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Basic-default.txt_/opt.yql +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Basic-default.txt_/opt.yql @@ -4,7 +4,7 @@ (let $3 (DataType 'String)) (let $4 (StructType '('"labels" (DictType $3 $3)) '('"ts" (DataType 'Datetime)) '('type $3) '('"value" (OptionalType (DataType 'Double))))) (let $5 '('"labels" '"value" '"ts" 'type)) -(let $6 (SoSourceSettings world '"my_project" (SecureParam '"cluster:default_local_solomon") $4 $5 '() '"1970-01-01T00:00:01Z" '"1970-01-02T00:00:01Z" '"" '"{}" (Bool '"false") '"AVG" '"PREVIOUS" (Uint32 '"15") '() '"")) +(let $6 (SoSourceSettings world '"my_project" (SecureParam '"cluster:default_local_solomon") $4 $5 '() '"1970-01-01T00:00:01Z" '"1970-01-02T00:00:01Z" '"" '"{}" (Bool '"false") '"AVG" '"PREVIOUS" (Uint32 '"15") '() '"" '())) (let $7 (DqStage '((DqSource (DataSource '"solomon" '"local_solomon") $6)) (lambda '($10) $10) '('('"_logical_id" '0)))) (let $8 (DqStage '((DqCnUnionAll (TDqOutput $7 '"0"))) (lambda '($11) $11) '('('"_logical_id" '0)))) (let $9 (ResPull! $1 $2 (Key) (DqCnResult (TDqOutput $8 '"0") '()) '('('type) '('autoref)) '"dq")) diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-BasicExtractMembers-default.txt_/opt.yql b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-BasicExtractMembers-default.txt_/opt.yql index 2e86037391c0..90bb5bcb4498 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-BasicExtractMembers-default.txt_/opt.yql +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-BasicExtractMembers-default.txt_/opt.yql @@ -2,7 +2,7 @@ (let $1 (Configure! world (DataSource '"config") '"DqEngine" '"force")) (let $2 (DataSink 'result)) (let $3 '('"value")) -(let $4 (SoSourceSettings world '"my_project" (SecureParam '"cluster:default_local_solomon") (StructType '('"value" (OptionalType (DataType 'Double)))) $3 '() '"1970-01-01T00:00:01Z" '"1970-01-02T00:00:01Z" '"" '"{}" (Bool '"false") '"AVG" '"PREVIOUS" (Uint32 '"15") '() '"")) +(let $4 (SoSourceSettings world '"my_project" (SecureParam '"cluster:default_local_solomon") (StructType '('"value" (OptionalType (DataType 'Double)))) $3 '() '"1970-01-01T00:00:01Z" '"1970-01-02T00:00:01Z" '"" '"{}" (Bool '"false") '"AVG" '"PREVIOUS" (Uint32 '"15") '() '"" '())) (let $5 (DqStage '((DqSource (DataSource '"solomon" '"local_solomon") $4)) (lambda '($9) $9) '('('"_logical_id" '0)))) (let $6 (DqStage '((DqCnUnionAll (TDqOutput $5 '"0"))) (lambda '($10) $10) '('('"_logical_id" '0)))) (let $7 '('('type) '('autoref) '('columns $3))) diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Downsampling-default.txt_/opt.yql b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Downsampling-default.txt_/opt.yql index 785e0de42ac9..96329416dc9b 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Downsampling-default.txt_/opt.yql +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Downsampling-default.txt_/opt.yql @@ -4,7 +4,7 @@ (let $3 (DataType 'String)) (let $4 (StructType '('"labels" (DictType $3 $3)) '('"ts" (DataType 'Datetime)) '('type $3) '('"value" (OptionalType (DataType 'Double))))) (let $5 '('"labels" '"value" '"ts" 'type)) -(let $6 (SoSourceSettings world '"my_project" (SecureParam '"cluster:default_local_solomon") $4 $5 '() '"1970-01-01T00:00:01Z" '"1970-01-02T00:00:01Z" '"" '"{}" (Bool '"false") '"SUM" '"PREVIOUS" (Uint32 '"25") '() '"")) +(let $6 (SoSourceSettings world '"my_project" (SecureParam '"cluster:default_local_solomon") $4 $5 '() '"1970-01-01T00:00:01Z" '"1970-01-02T00:00:01Z" '"" '"{}" (Bool '"false") '"SUM" '"PREVIOUS" (Uint32 '"25") '() '"" '())) (let $7 (DqStage '((DqSource (DataSource '"solomon" '"local_solomon") $6)) (lambda '($10) $10) '('('"_logical_id" '0)))) (let $8 (DqStage '((DqCnUnionAll (TDqOutput $7 '"0"))) (lambda '($11) $11) '('('"_logical_id" '0)))) (let $9 (ResPull! $1 $2 (Key) (DqCnResult (TDqOutput $8 '"0") '()) '('('type) '('autoref)) '"dq")) diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-DownsamplingValidSettings-default.txt_/opt.yql b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-DownsamplingValidSettings-default.txt_/opt.yql index 372609f24f4b..247284dd9f3e 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-DownsamplingValidSettings-default.txt_/opt.yql +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-DownsamplingValidSettings-default.txt_/opt.yql @@ -4,7 +4,7 @@ (let $3 (DataType 'String)) (let $4 (StructType '('"labels" (DictType $3 $3)) '('"ts" (DataType 'Datetime)) '('type $3) '('"value" (OptionalType (DataType 'Double))))) (let $5 '('"labels" '"value" '"ts" 'type)) -(let $6 (SoSourceSettings world '"my_project" (SecureParam '"cluster:default_local_solomon") $4 $5 '() '"1970-01-01T00:00:01Z" '"1970-01-02T00:00:01Z" '"" '"{}" (Bool '"false") '"SUM" '"PREVIOUS" (Uint32 '"15") '() '"")) +(let $6 (SoSourceSettings world '"my_project" (SecureParam '"cluster:default_local_solomon") $4 $5 '() '"1970-01-01T00:00:01Z" '"1970-01-02T00:00:01Z" '"" '"{}" (Bool '"false") '"SUM" '"PREVIOUS" (Uint32 '"15") '() '"" '())) (let $7 (DqStage '((DqSource (DataSource '"solomon" '"local_solomon") $6)) (lambda '($10) $10) '('('"_logical_id" '0)))) (let $8 (DqStage '((DqCnUnionAll (TDqOutput $7 '"0"))) (lambda '($11) $11) '('('"_logical_id" '0)))) (let $9 (ResPull! $1 $2 (Key) (DqCnResult (TDqOutput $8 '"0") '()) '('('type) '('autoref)) '"dq")) diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-HistResponse-default.txt_/opt.yql b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-HistResponse-default.txt_/opt.yql index a791d053cc76..edf2c4d43f40 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-HistResponse-default.txt_/opt.yql +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-HistResponse-default.txt_/opt.yql @@ -4,7 +4,7 @@ (let $3 (DataType 'String)) (let $4 (StructType '('"labels" (DictType $3 $3)) '('"ts" (DataType 'Datetime)) '('type $3) '('"value" (OptionalType (DataType 'Double))))) (let $5 '('"labels" '"value" '"ts" 'type)) -(let $6 (SoSourceSettings world '"hist" (SecureParam '"cluster:default_local_solomon") $4 $5 '() '"1970-01-01T00:00:01Z" '"1970-01-02T00:00:01Z" '"" '"histogram_percentile(95, {})" (Bool '"false") '"AVG" '"PREVIOUS" (Uint32 '"15") '() '"")) +(let $6 (SoSourceSettings world '"hist" (SecureParam '"cluster:default_local_solomon") $4 $5 '() '"1970-01-01T00:00:01Z" '"1970-01-02T00:00:01Z" '"" '"histogram_percentile(95, {})" (Bool '"false") '"AVG" '"PREVIOUS" (Uint32 '"15") '() '"" '())) (let $7 (DqStage '((DqSource (DataSource '"solomon" '"local_solomon") $6)) (lambda '($10) $10) '('('"_logical_id" '0)))) (let $8 (DqStage '((DqCnUnionAll (TDqOutput $7 '"0"))) (lambda '($11) $11) '('('"_logical_id" '0)))) (let $9 (ResPull! $1 $2 (Key) (DqCnResult (TDqOutput $8 '"0") '()) '('('type) '('autoref)) '"dq")) diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-LabelColumns-default.txt_/opt.yql b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-LabelColumns-default.txt_/opt.yql index c0428ab6ec95..fd7a723e1333 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-LabelColumns-default.txt_/opt.yql +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-LabelColumns-default.txt_/opt.yql @@ -8,7 +8,7 @@ (let $7 (StructType '($3 $4) '($5 $4) '($6 $4) '('"project" $4) '('"ts" (DataType 'Datetime)) '('type $4) '('"value" (OptionalType (DataType 'Double))))) (let $8 '('"value" '"ts" 'type)) (let $9 '($3 $5 '"project" $6)) -(let $10 (SoSourceSettings world '"my_project" (SecureParam '"cluster:default_local_solomon") $7 $8 $9 '"1970-01-01T00:00:01Z" '"1970-01-02T00:00:01Z" '"" '"{}" (Bool '"false") '"AVG" '"PREVIOUS" (Uint32 '"15") '() '"")) +(let $10 (SoSourceSettings world '"my_project" (SecureParam '"cluster:default_local_solomon") $7 $8 $9 '"1970-01-01T00:00:01Z" '"1970-01-02T00:00:01Z" '"" '"{}" (Bool '"false") '"AVG" '"PREVIOUS" (Uint32 '"15") '() '"" $9)) (let $11 (DqStage '((DqSource (DataSource '"solomon" '"local_solomon") $10)) (lambda '($14) $14) '('('"_logical_id" '0)))) (let $12 (DqStage '((DqCnUnionAll (TDqOutput $11 '"0"))) (lambda '($15) $15) '('('"_logical_id" '0)))) (let $13 (ResPull! $1 $2 (Key) (DqCnResult (TDqOutput $12 '"0") '()) '('('type) '('autoref)) '"dq")) diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Subquery-default.txt_/opt.yql b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Subquery-default.txt_/opt.yql index 1bd756e5779b..d8f4eb762718 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Subquery-default.txt_/opt.yql +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Subquery-default.txt_/opt.yql @@ -4,7 +4,7 @@ (let $3 (DataType 'String)) (let $4 (StructType '('"labels" (DictType $3 $3)) '('"ts" (DataType 'Datetime)) '('type $3) '('"value" (OptionalType (DataType 'Double))))) (let $5 '('"labels" '"value" '"ts" 'type)) -(let $6 (SoSourceSettings world '"my_project" (SecureParam '"cluster:default_local_solomon") $4 $5 '() '"1970-01-01T00:00:01Z" '"1970-01-02T00:00:01Z" '"" '"{}" (Bool '"false") '"SUM" '"PREVIOUS" (Uint32 '"25") '() '"")) +(let $6 (SoSourceSettings world '"my_project" (SecureParam '"cluster:default_local_solomon") $4 $5 '() '"1970-01-01T00:00:01Z" '"1970-01-02T00:00:01Z" '"" '"{}" (Bool '"false") '"SUM" '"PREVIOUS" (Uint32 '"25") '() '"" '())) (let $7 (DqStage '((DqSource (DataSource '"solomon" '"local_solomon") $6)) (lambda '($11) $11) '('('"_logical_id" '0)))) (let $8 (DqStage '((DqCnUnionAll (TDqOutput $7 '"0"))) (lambda '($12) $12) '('('"_logical_id" '0)))) (let $9 '('('type) '('autoref) '('unordered)))