From e1d0b6db730de2fb1d7dc8382037df7910be3bc4 Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Fri, 17 Oct 2025 07:29:04 +0000 Subject: [PATCH 01/43] Add ForceShuffleElimination option to use for benchmarking --- ydb/core/kqp/opt/logical/kqp_opt_log.cpp | 2 +- ydb/library/yql/dq/opt/dq_opt_join_cbo_factory.cpp | 2 +- ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp | 6 +++--- ydb/library/yql/dq/opt/dq_opt_join_cost_based.h | 2 +- ydb/library/yql/dq/opt/ut/dq_cbo_ut.cpp | 3 +-- yql/essentials/core/cbo/cbo_optimizer_new.h | 6 +++--- yql/essentials/core/cbo/simple/cbo_simple.cpp | 2 +- 7 files changed, 11 insertions(+), 12 deletions(-) diff --git a/ydb/core/kqp/opt/logical/kqp_opt_log.cpp b/ydb/core/kqp/opt/logical/kqp_opt_log.cpp index 7fa5c7e71912..0b76357c8591 100644 --- a/ydb/core/kqp/opt/logical/kqp_opt_log.cpp +++ b/ydb/core/kqp/opt/logical/kqp_opt_log.cpp @@ -175,7 +175,7 @@ class TKqpLogicalOptTransformer : public TOptimizeTransformerBase { } TMaybeNode OptimizeEquiJoinWithCosts(TExprBase node, TExprContext& ctx) { - TCBOSettings settings { + TOptimizerSettings settings { .MaxDPhypDPTableSize = Config->MaxDPHypDPTableSize.Get().GetOrElse(TDqSettings::TDefault::MaxDPHypDPTableSize), .ShuffleEliminationJoinNumCutoff = Config->ShuffleEliminationJoinNumCutoff.Get().GetOrElse(TDqSettings::TDefault::ShuffleEliminationJoinNumCutoff) }; diff --git a/ydb/library/yql/dq/opt/dq_opt_join_cbo_factory.cpp b/ydb/library/yql/dq/opt/dq_opt_join_cbo_factory.cpp index 46f6bba25420..e19bb3f6b9f0 100644 --- a/ydb/library/yql/dq/opt/dq_opt_join_cbo_factory.cpp +++ b/ydb/library/yql/dq/opt/dq_opt_join_cbo_factory.cpp @@ -9,7 +9,7 @@ namespace NYql::NDq { namespace { class TDqOptimizerFactory : public IOptimizerFactory { public: - virtual IOptimizerNew::TPtr MakeJoinCostBasedOptimizerNative(IProviderContext& pctx, TExprContext& ectx, const TCBOSettings& settings) const override { + virtual IOptimizerNew::TPtr MakeJoinCostBasedOptimizerNative(IProviderContext& pctx, TExprContext& ectx, const TOptimizerSettings& settings) const override { return IOptimizerNew::TPtr(MakeNativeOptimizerNew(pctx, settings, ectx, false, nullptr)); } diff --git a/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp b/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp index 8029d2455361..53c581c8ebf0 100644 --- a/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp +++ b/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp @@ -314,7 +314,7 @@ class TOptimizerNativeNew: public IOptimizerNew { public: TOptimizerNativeNew( IProviderContext& ctx, - const TCBOSettings &optimizerSettings, + const TOptimizerSettings &optimizerSettings, TExprContext& exprCtx, bool enableShuffleElimination, TSimpleSharedPtr orderingsFSM, @@ -506,7 +506,7 @@ class TOptimizerNativeNew: public IOptimizerNew { } private: - TCBOSettings OptimizerSettings_; + TOptimizerSettings OptimizerSettings_; TExprContext& ExprCtx; bool EnableShuffleElimination; @@ -516,7 +516,7 @@ class TOptimizerNativeNew: public IOptimizerNew { IOptimizerNew* MakeNativeOptimizerNew( IProviderContext& pctx, - const TCBOSettings &settings, + const TOptimizerSettings &settings, TExprContext& ectx, bool enableShuffleElimination, TSimpleSharedPtr orderingsFSM, diff --git a/ydb/library/yql/dq/opt/dq_opt_join_cost_based.h b/ydb/library/yql/dq/opt/dq_opt_join_cost_based.h index 9ba43bceee26..448d1dbb2a0d 100644 --- a/ydb/library/yql/dq/opt/dq_opt_join_cost_based.h +++ b/ydb/library/yql/dq/opt/dq_opt_join_cost_based.h @@ -50,7 +50,7 @@ void CollectInterestingOrderingsFromJoinTree( IOptimizerNew* MakeNativeOptimizerNew( IProviderContext& ctx, - const TCBOSettings &settings, + const TOptimizerSettings &settings, TExprContext& ectx, bool enableShuffleElimination, TSimpleSharedPtr orderingsFSM = nullptr, diff --git a/ydb/library/yql/dq/opt/ut/dq_cbo_ut.cpp b/ydb/library/yql/dq/opt/ut/dq_cbo_ut.cpp index 6b08d5e9cc24..3c8b700e5228 100644 --- a/ydb/library/yql/dq/opt/ut/dq_cbo_ut.cpp +++ b/ydb/library/yql/dq/opt/ut/dq_cbo_ut.cpp @@ -378,8 +378,7 @@ Y_UNIT_TEST(DqOptimizeEquiJoinWithCostsNative) { TExprContext ctx; TBaseProviderContext pctx; std::function optFactory = [&]() { - TCBOSettings settings{}; - return MakeNativeOptimizerNew(pctx, settings, ctx, false); + return MakeNativeOptimizerNew(pctx, TOptimizerSettings{.MaxDPHypDPTableSize = 100000}, ctx, false); }; _DqOptimizeEquiJoinWithCosts(optFactory, ctx); } diff --git a/yql/essentials/core/cbo/cbo_optimizer_new.h b/yql/essentials/core/cbo/cbo_optimizer_new.h index 232ebd9addae..362be2720bfc 100644 --- a/yql/essentials/core/cbo/cbo_optimizer_new.h +++ b/yql/essentials/core/cbo/cbo_optimizer_new.h @@ -385,19 +385,19 @@ struct IOptimizerNew { struct TExprContext; -struct TCBOSettings { +struct TOptimizerSettings { ui32 MaxDPhypDPTableSize = 100000; ui32 ShuffleEliminationJoinNumCutoff = 14; }; -class IOptimizerFactory: private TNonCopyable { +class IOptimizerFactory : private TNonCopyable { public: using TPtr = std::shared_ptr; using TLogger = std::function; virtual ~IOptimizerFactory() = default; - virtual IOptimizerNew::TPtr MakeJoinCostBasedOptimizerNative(IProviderContext& pctx, TExprContext& ctx, const TCBOSettings& settings) const = 0; + virtual IOptimizerNew::TPtr MakeJoinCostBasedOptimizerNative(IProviderContext& pctx, TExprContext& ctx, const TOptimizerSettings& settings) const = 0; struct TPGSettings { TLogger Logger = [](const TString&) {}; diff --git a/yql/essentials/core/cbo/simple/cbo_simple.cpp b/yql/essentials/core/cbo/simple/cbo_simple.cpp index 0543c0ab7923..e729626283e3 100644 --- a/yql/essentials/core/cbo/simple/cbo_simple.cpp +++ b/yql/essentials/core/cbo/simple/cbo_simple.cpp @@ -9,7 +9,7 @@ namespace { class TSimpleOptimizerFactory: public IOptimizerFactory { public: - virtual IOptimizerNew::TPtr MakeJoinCostBasedOptimizerNative(IProviderContext& pctx, TExprContext& ctx, const TCBOSettings& settings) const override { + virtual IOptimizerNew::TPtr MakeJoinCostBasedOptimizerNative(IProviderContext& pctx, TExprContext& ctx, const TOptimizerSettings& settings) const override { Y_UNUSED(pctx); Y_UNUSED(ctx); Y_UNUSED(settings); From edc9eeb4bd3447ddfe181051506917eedcb0d43f Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Fri, 17 Oct 2025 07:29:53 +0000 Subject: [PATCH 02/43] Fix PRAGMA message for MaxDPHypDPTableSize --- ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp b/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp index 53c581c8ebf0..50d66c083133 100644 --- a/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp +++ b/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp @@ -388,7 +388,7 @@ class TOptimizerNativeNew: public IOptimizerNew { YqlIssue( {}, TIssuesIds::CBO_ENUM_LIMIT_REACHED, "Cost Based Optimizer could not be applied to this query: " - "Enumeration is too large, use PRAGMA MaxDPHypDPTableSize='4294967295' to disable the limitation" + "Enumeration is too large, use PRAGMA ydb.MaxDPHypDPTableSize='4294967295' to disable the limitation" ) ); ComputeStatistics(joinTree, this->Pctx); From 6c3a78ed997e32ddfda749e8c7c3786b0fa45991 Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Fri, 17 Oct 2025 08:51:34 +0000 Subject: [PATCH 03/43] Add join order tests on different randomized topologies --- ydb/core/kqp/ut/join/kqp_join_order_ut.cpp | 160 +++++++++ .../ut/join/kqp_join_topology_generator.cpp | 332 ++++++++++++++++++ .../kqp/ut/join/kqp_join_topology_generator.h | 126 +++++++ ydb/core/kqp/ut/join/ya.make | 1 + 4 files changed, 619 insertions(+) create mode 100644 ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp create mode 100644 ydb/core/kqp/ut/join/kqp_join_topology_generator.h diff --git a/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp index 2887e80086f7..f4620e31b7ee 100644 --- a/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp @@ -1,6 +1,8 @@ #include #include +#include +#include #include #include @@ -9,6 +11,9 @@ #include #include +#include "kqp_join_topology_generator.h" + + namespace NKikimr { namespace NKqp { @@ -613,6 +618,145 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) { } } + TString ExplainQuery(NYdb::NQuery::TSession session, const std::string &query) { + auto explainRes = session.ExecuteQuery(query, + NYdb::NQuery::TTxControl::NoTx(), + NYdb::NQuery::TExecuteQuerySettings().ExecMode(NQuery::EExecMode::Explain) + ).ExtractValueSync(); + + Cout << query << "\n"; + explainRes.GetIssues().PrintTo(Cout); + Cout << "Status: " << explainRes.IsSuccess() << "\n"; + UNIT_ASSERT_VALUES_EQUAL(explainRes.GetStatus(), EStatus::SUCCESS); + + return *explainRes.GetStats()->GetPlan(); + } + + std::vector ExecuteQuery(NYdb::NQuery::TSession session, std::string query) { + auto execRes = session.ExecuteQuery(query, NYdb::NQuery::TTxControl::NoTx()).ExtractValueSync(); + + Cout << query << "\n"; + execRes.GetIssues().PrintTo(Cout); + Cout << "Status: " << execRes.IsSuccess() << "\n"; + UNIT_ASSERT(execRes.IsSuccess()); + + return execRes.GetResultSets(); + } + + void JustPrintPlan(const TString &plan) { + NYdb::NConsoleClient::TQueryPlanPrinter queryPlanPrinter( + NYdb::NConsoleClient::EDataFormat::PrettyTable, + /*analyzeMode=*/true, Cout, /*maxWidth=*/0 + ); + + queryPlanPrinter.Print(plan); + } + + std::string ConfigureQuery(const std::string &query, bool enableShuffleElimination = false, unsigned optLevel = 2) { + std::string queryWithShuffleElimination = "PRAGMA ydb.OptShuffleElimination = \""; + queryWithShuffleElimination += enableShuffleElimination ? "true" : "false"; + queryWithShuffleElimination += "\";\n"; + queryWithShuffleElimination += "PRAGMA ydb.MaxDPHypDPTableSize='4294967295';\n"; + queryWithShuffleElimination += "PRAGMA ydb.ForceShuffleElimination='true';\n"; + queryWithShuffleElimination += "PRAGMA ydb.CostBasedOptimizationLevel = \"" + std::to_string(optLevel) + "\";\n"; + queryWithShuffleElimination += query; + + return queryWithShuffleElimination; + } + + struct TShuffleEliminationBenchResult { + unsigned TimeWithShuffleEliminationNanos = 0; + unsigned TimeWithoutShuffleEliminationNanos = 0; + }; + + TShuffleEliminationBenchResult BenchmarkShuffleElimination(NYdb::NQuery::TSession session, const TString& query) { + namespace Nsc = std::chrono; + using TClock = Nsc::high_resolution_clock; + + TString queryWithoutCBO = ConfigureQuery(query, /*enableShuffleElimination=*/false, /*optLevel=*/1); + unsigned nanosWithoutCBO = 0; + { + std::chrono::time_point start = TClock::now(); + ExplainQuery(session, queryWithoutCBO); + nanosWithoutCBO = Nsc::duration_cast(TClock::now() - start).count(); + Cout << "Nanoseconds for query without CBO: " << nanosWithoutCBO << "\n"; + } + + TString queryWithShuffleElimination = ConfigureQuery(query, /*enableShuffleElimination=*/true); + TString planWithShuffleElimnation; + unsigned nanosWith = 0; + { + + std::chrono::time_point start = TClock::now(); + planWithShuffleElimnation = ExplainQuery(session, queryWithShuffleElimination); + nanosWith = Nsc::duration_cast(TClock::now() - start).count(); + Cout << "Nanoseconds for CBO with shuffle elimination: " << nanosWith - nanosWithoutCBO << "\n"; + } + + TString queryWithoutShuffleElimination = ConfigureQuery(query, /*enableShuffleElimination=*/false); + TString planWithoutShuffleElimnation; + unsigned nanosWithout = 0; + { + std::chrono::time_point start = TClock::now(); + planWithoutShuffleElimnation = ExplainQuery(session, queryWithoutShuffleElimination); + nanosWithout = Nsc::duration_cast(TClock::now() - start).count(); + Cout << "Nanoseconds for CBO without shuffle elimination: " << nanosWithout - nanosWithoutCBO << "\n"; + } + + JustPrintPlan(planWithShuffleElimnation); + JustPrintPlan(planWithoutShuffleElimnation); + + return { nanosWith, nanosWithout }; + } + + struct TTopologyShuffleEliminationResult { + double NewColumnProbability = 0.0; + int NumTables = 0; + TShuffleEliminationBenchResult Bench = {}; + + void Dump(IOutputStream &OS) const { + OS << NewColumnProbability << " " << NumTables << " " + << Bench.TimeWithShuffleEliminationNanos << " " + << Bench.TimeWithoutShuffleEliminationNanos << "\n"; + } + }; + + template + void BenchmarkShuffleEliminationOnTopology(int maxNumTables, double probabilityStep, unsigned seed = 0) { + std::mt19937 mt(seed); + + TSchema fullSchema = TSchema::MakeWithEnoughColumns(maxNumTables); + TString stats = TSchemaStats::MakeRandom(mt, fullSchema, 7, 10).ToJSON(); + + auto kikimr = GetKikimrWithJoinSettings(/*useStreamLookupJoin=*/false, /*stats=*/stats, /*useCBO=*/true, TExecuteParams{}); + auto db = kikimr.GetQueryClient(); + auto session = db.GetSession().GetValueSync().GetSession(); + + std::vector data; + int totalCount = 0; + for (double probability = 0; probability <= 1.0; probability += probabilityStep) { + for (int i = 2 /* one table is not possible with all topoloties, so 2 */; i < maxNumTables; i ++) { + TRelationGraph graph = Lambda(mt, i, 0.5); + + ExecuteQuery(session, graph.GetSchema().MakeCreateQuery()); + + auto resultTime = BenchmarkShuffleElimination(session, graph.MakeQuery()); + data.push_back({probability, i, resultTime}); + + Cout << "totalCount: " << ++ totalCount << "\n"; + Cout << "result: "; + data.back().Dump(Cout); + + ExecuteQuery(session, graph.GetSchema().MakeDropQuery()); + } + } + + // Printing a report of all measurements + for (const auto &result: data) { + result.Dump(Cout); + } + } + void CheckJoinCardinality(const TString& queryPath, const TString& statsPath, const TString& joinKind, double card, bool useStreamLookupJoin, bool useColumnStore) { auto kikimr = GetKikimrWithJoinSettings(useStreamLookupJoin, GetStatic(statsPath), true); kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableViews(true); @@ -863,6 +1007,22 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) { UNIT_ASSERT_C(join.Join == "InnerJoin (Grace)", join.Join); } + Y_UNIT_TEST(ShuffleEliminationBenchmarkOnStar) { + BenchmarkShuffleEliminationOnTopology(/*maxNumTables=*/15, /*probabilityStep=*/0.2); + } + + Y_UNIT_TEST(ShuffleEliminationBenchmarkOnLine) { + BenchmarkShuffleEliminationOnTopology(/*maxNumTables=*/15, /*probabilityStep=*/0.2); + } + + Y_UNIT_TEST(ShuffleEliminationBenchmarkOnFullyConnected) { + BenchmarkShuffleEliminationOnTopology(/*maxNumTables=*/15, /*probabilityStep=*/0.2); + } + + Y_UNIT_TEST(ShuffleEliminationBenchmarkOnRandomTree) { + BenchmarkShuffleEliminationOnTopology(/*maxNumTables=*/15, /*probabilityStep=*/0.2); + } + Y_UNIT_TEST_TWIN(ShuffleEliminationOneJoin, EnableSeparationComputeActorsFromRead) { auto [plan, _] = ExecuteJoinOrderTestGenericQueryWithStats("queries/shuffle_elimination_one_join.sql", "stats/tpch1000s.json", false, true, true, {.EnableSeparationComputeActorsFromRead = EnableSeparationComputeActorsFromRead}); auto joinFinder = TFindJoinWithLabels(plan); diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp new file mode 100644 index 000000000000..e0a8964a29ae --- /dev/null +++ b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp @@ -0,0 +1,332 @@ +#include "kqp_join_topology_generator.h" + +#include +#include +#include +#include +#include +#include + + +namespace NKikimr::NKqp { + +class TLexicographicalNameGenerator { +public: + static std::string getName(unsigned ID, bool lowerCase = true) { + if (ID < Base_) + return std::string(1, fromDigit(ID, lowerCase)); + + ID -= Base_; + + unsigned count = 1; + unsigned step = Base_; + for (; ID >= step;) { + ID -= step; + step *= step; + count *= 2; + } + + std::string result(count, fromDigit(Base_ - 1, lowerCase)); + return result + fromNumber(ID, result.size(), lowerCase); + } + +private: + static std::string fromNumber(unsigned number, unsigned size, bool lowerCase) { + std::string stringified = ""; + for (unsigned i = 0; i < size; ++ i) { + stringified.push_back(fromDigit(number % Base_, lowerCase)); + number /= Base_; + } + + return std::string(stringified.rbegin(), stringified.rend()); + } + + static char fromDigit(unsigned value, bool lowerCase) { + assert(0 <= value && value < Base_); + return (lowerCase ? 'a' : 'A') + value; + } + + static constexpr unsigned Base_ = 'z' - 'a' + 1; +}; + + +static std::string getTableName(unsigned tableID) { + return TLexicographicalNameGenerator::getName(tableID, /*lowerCase=*/false); +} + +static std::string getColumnName(unsigned tableID, unsigned columnID) { + return TLexicographicalNameGenerator::getName(tableID, /*lowerCase=*/true) + "_" + + TLexicographicalNameGenerator::getName(columnID, /*lowerCase=*/true); +} + +static std::string getRelationName(unsigned tableID, unsigned columnID) { + return getTableName(tableID) + "." + getColumnName(tableID, columnID); +} + +static std::string getTablePath(unsigned tableID) { + return "/Root/" + getTableName(tableID); +} + + +static unsigned getRandom(std::mt19937 &mt, unsigned a, unsigned b) { + std::uniform_int_distribution<> distribution(a, b); + return distribution(mt); +} + +static unsigned getRandomBool(std::mt19937 &mt, double trueProbability) { + std::uniform_real_distribution<> distribution(0.0, 1.0); + return distribution(mt) < trueProbability; +} + + +unsigned TTable::GetRandomOrNewColumn(std::mt19937 &mt, double newColumnProbability) { + bool generateNewColumn = getRandomBool(mt, newColumnProbability); + if (generateNewColumn || NumColumns == 0) { + return NumColumns ++; + } + + return GetRandomColumn(mt); +} + +unsigned TTable::GetRandomColumn(std::mt19937 &mt) const { + assert(NumColumns != 0); + return getRandom(mt, 0, NumColumns - 1); +} + + +TSchema TSchema::MakeWithEnoughColumns(unsigned numNodes) { + return TSchema{std::vector(numNodes, TTable{numNodes})}; +} + +std::string TSchema::MakeCreateQuery() const { + std::string prerequisites; + for (unsigned i = 0; i < Tables_.size(); ++ i) { + prerequisites += "CREATE TABLE `" + getTablePath(i) + "` (\n"; + + for (unsigned j = 0; j < Tables_[i].GetNumColumns(); ++ j) { + prerequisites += " " + getColumnName(i, j) + " Int32 NOT NULL,\n"; + } + prerequisites += " PRIMARY KEY (" + getColumnName(i, 0) + ")\n"; + + prerequisites += ") WITH (STORE = COLUMN);\n"; + } + + return prerequisites; +} + +std::string TSchema::MakeDropQuery() const { + std::string query; + for (unsigned i = 0; i < Tables_.size(); ++ i) { + query += "DROP TABLE `" + getTablePath(i) + "`;\n"; + } + + return query; +} + + +TRelationGraph TRelationGraph::FromPrufer(std::mt19937 &mt, const std::vector& prufer, double newColumnProbability) { + unsigned n = prufer.size() + 2; + + std::vector degree(n, 1); + for (unsigned i : prufer) { + ++ degree[i]; + } + + TRelationGraph graph(n); + + for (unsigned u : prufer) { + for (unsigned v = 0; v < n; ++ v) { + if (degree[v] == 1) { + graph.Connect(mt, u, v, newColumnProbability); + + -- degree[v]; + -- degree[u]; + break; + } + } + } + + int u = -1; + unsigned v = 0; + for (; v < n; ++ v) { + if (degree[v] == 1) { + if (u != -1) { + graph.Connect(mt, u, v, newColumnProbability); + break; + } + + u = v; + } + } + + return graph; +} + +void TRelationGraph::Connect(std::mt19937 &mt, unsigned lhs, unsigned rhs, double newColumnProbability) { + unsigned ColumnLHS = Schema_[lhs].GetRandomOrNewColumn(mt, newColumnProbability); + unsigned ColumnRHS = Schema_[rhs].GetRandomOrNewColumn(mt, newColumnProbability); + + AdjacencyList_[lhs].push_back({rhs, ColumnLHS, ColumnRHS}); + AdjacencyList_[rhs].push_back({lhs, ColumnRHS, ColumnLHS}); +} + +std::string TRelationGraph::MakeQuery() const { + std::string fromClause; + std::string joinClause; + for (unsigned i = 0; i < AdjacencyList_.size(); ++ i) { + std::string currentJoin = + "JOIN " + TLexicographicalNameGenerator::getName(i, /*lowerCase=*/false) + " ON"; + + bool hasJoin = false; + for (unsigned j = 0; j < AdjacencyList_[i].size(); ++ j) { + const unsigned connectionsJ = AdjacencyList_[AdjacencyList_[i][j].Target].size(); + + if (AdjacencyList_[i].size() > connectionsJ) { + continue; + } + + if (AdjacencyList_[i].size() == connectionsJ && i < AdjacencyList_[i][j].Target) { + continue; + } + + if (hasJoin) { + currentJoin += " AND"; + } + + hasJoin = true; + + currentJoin += " " + + getRelationName(i, AdjacencyList_[i][j].ColumnLHS) + " = " + + getRelationName(AdjacencyList_[i][j].Target, AdjacencyList_[i][j].ColumnRHS); + } + currentJoin += "\n"; + + if (hasJoin) { + joinClause += currentJoin; + continue; + } else { + assert(fromClause.empty()); + fromClause = "SELECT *\nFROM " + getTableName(i) + "\n"; + } + } + + // remove extra '\n' from last join + if (!joinClause.empty()) { + assert(joinClause.back() == '\n'); + joinClause.pop_back(); + } + + return std::move(fromClause) + std::move(joinClause) + ";\n"; +} + +void TRelationGraph::DumpGraph(std::ostream &OS) const { + OS << "graph {\n"; + for (unsigned i = 0; i < AdjacencyList_.size(); ++ i) { + for (unsigned j = 0; j < AdjacencyList_[i].size(); ++ j) { + const TEdge &edge = AdjacencyList_[i][j]; + if (i <= edge.Target) { + OS << " " << getTableName(i) << " -- " << getTableName(edge.Target) + << " [label = \"" + << getRelationName(i, edge.ColumnLHS) << " = " + << getRelationName(edge.Target, edge.ColumnRHS) + << "\"];\n"; + } + } + } + + OS << "}\n"; +} + + +TSchemaStats TSchemaStats::MakeRandom(std::mt19937 &mt, const TSchema &schema, unsigned a, unsigned b) { + std::uniform_int_distribution<> distribution(a, b); + std::vector stats(schema.GetSize()); + + for (unsigned i = 0; i < schema.GetSize(); ++ i) { + unsigned RowSize = std::pow(10, distribution(mt)); + unsigned ByteSize = RowSize * 64; + stats[i] = {ByteSize, RowSize}; + } + + return TSchemaStats{stats}; +} + +std::string TSchemaStats::ToJSON() const { + std::stringstream ss; + + ss << "{"; + for (unsigned i = 0; i < Stats_.size(); ++ i) { + if (i != 0) + ss << ","; + + ss << "\"/Root/" << TLexicographicalNameGenerator::getName(i, /*lowerCase=*/false) << "\": "; + ss << "{"; + ss << "\"" << "n_rows" << "\": " << Stats_[i].RowSize << ", "; + ss << "\"" << "byte_size" << "\": " << Stats_[i].ByteSize; + ss << "}"; + } + ss << "}"; + + return ss.str(); +} + +TRelationGraph GenerateLine(std::mt19937 &mt, unsigned numNodes, double newColumnProbability) { + TRelationGraph graph(numNodes); + + unsigned lastVertex = 0; + bool first = true; + for (unsigned i = 0; i < numNodes; ++ i) { + if (!first) { + graph.Connect(mt, lastVertex, i, newColumnProbability); + } + + first = false; + lastVertex = i; + } + + return graph; +} + +TRelationGraph GenerateStar(std::mt19937 &mt, unsigned numNodes, double newColumnProbability) { + TRelationGraph graph(numNodes); + + unsigned root = 0; + for (unsigned i = 1; i < numNodes; ++ i) { + graph.Connect(mt, root, i, newColumnProbability); + } + + return graph; +} + +TRelationGraph GenerateFullyConnected(std::mt19937 &mt, unsigned numNodes, + double newColumnProbability) { + + TRelationGraph graph(numNodes); + + for (unsigned i = 0; i < numNodes; ++ i) { + for (unsigned j = i + 1; j < numNodes; ++ j) { + graph.Connect(mt, i, j, newColumnProbability); + } + } + + return graph; +} + +static std::vector GenerateRandomPruferSequence(std::mt19937 &mt, unsigned numNodes) { + assert(n >= 2); + std::uniform_int_distribution<> distribution(0, numNodes - 1); + + std::vector prufer(numNodes - 2); + for (unsigned i = 0; i < numNodes - 2; ++i) { + prufer[i] = distribution(mt); + } + + return prufer; +} + +TRelationGraph GenerateRandomTree(std::mt19937 &mt, unsigned numNodes, double newColumnProbability) { + auto prufer = GenerateRandomPruferSequence(mt, numNodes); + return TRelationGraph::FromPrufer(mt, prufer, newColumnProbability); +} + +} diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_generator.h b/ydb/core/kqp/ut/join/kqp_join_topology_generator.h new file mode 100644 index 000000000000..f34bb34deb35 --- /dev/null +++ b/ydb/core/kqp/ut/join/kqp_join_topology_generator.h @@ -0,0 +1,126 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + + +namespace NKikimr::NKqp { + +class TTable { +public: + TTable(unsigned numColumns = 0) + : NumColumns(numColumns) + { + } + + unsigned GetRandomOrNewColumn(std::mt19937 &mt, double newColumnProbability); + + unsigned GetNumColumns() const { + return NumColumns; + } + +private: + unsigned GetRandomColumn(std::mt19937 &mt) const; + +private: + // table has columns from 0..NumColumns + unsigned NumColumns; +}; + + +class TSchema { +public: + TSchema(unsigned numNodes) + : Tables_(numNodes) + { + } + + TSchema(std::vector tables) + : Tables_(std::move(tables)) + { + } + + static TSchema MakeWithEnoughColumns(unsigned numNodes); + + std::string MakeCreateQuery() const; + std::string MakeDropQuery() const; + + TTable& operator[](unsigned index) { + return Tables_[index]; + } + + size_t GetSize() const { + return Tables_.size(); + } + +private: + std::vector Tables_; +}; + + +class TRelationGraph { +public: + TRelationGraph(unsigned numNodes) + : AdjacencyList_(numNodes) + , Schema_(numNodes) + { + } + + static TRelationGraph FromPrufer(std::mt19937 &mt, const std::vector& prufer, double newColumnProbability); + + void Connect(std::mt19937 &mt, unsigned lhs, unsigned rhs, double newColumnProbability); + + std::string MakeQuery() const; + + void DumpGraph(std::ostream &OS) const; + + const TSchema& GetSchema() const { + return Schema_; + } + +private: + struct TEdge { + unsigned Target; + unsigned ColumnLHS, ColumnRHS; + }; + + using TAdjacencyList = std::vector>; + + TAdjacencyList AdjacencyList_; + TSchema Schema_; +}; + + +class TSchemaStats { +public: + struct TTableStats { + unsigned ByteSize; + unsigned RowSize; + }; + +public: + TSchemaStats(std::vector stats) + : Stats_(std::move(stats)) + { + } + + static TSchemaStats MakeRandom(std::mt19937 &mt, const TSchema &schema, unsigned a, unsigned b); + + std::string ToJSON() const; + +private: + std::vector Stats_; +}; + + +TRelationGraph GenerateLine(std::mt19937 &mt, unsigned numNodes, double newColumnProbability); +TRelationGraph GenerateStar(std::mt19937 &mt, unsigned numNodes, double newColumnProbability); +TRelationGraph GenerateFullyConnected(std::mt19937 &mt, unsigned numNodes, double newColumnProbability); +TRelationGraph GenerateRandomTree(std::mt19937 &mt, unsigned numNodes, double newColumnProbability); + +} // namespace NKikimr::NKqp + diff --git a/ydb/core/kqp/ut/join/ya.make b/ydb/core/kqp/ut/join/ya.make index 93e27c09c801..46c72a6e9621 100644 --- a/ydb/core/kqp/ut/join/ya.make +++ b/ydb/core/kqp/ut/join/ya.make @@ -24,6 +24,7 @@ SRCS( kqp_index_lookup_join_ut.cpp kqp_join_ut.cpp kqp_join_order_ut.cpp + kqp_join_topology_generator.cpp ) PEERDIR( From b1e62d3cc6a355520af12fc8a9c6600ab03cd140 Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Tue, 21 Oct 2025 15:58:58 +0000 Subject: [PATCH 04/43] Fix table appearing in join condition before its join clause --- .../ut/join/kqp_join_topology_generator.cpp | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp index e0a8964a29ae..f167eee4e546 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp @@ -178,17 +178,7 @@ std::string TRelationGraph::MakeQuery() const { "JOIN " + TLexicographicalNameGenerator::getName(i, /*lowerCase=*/false) + " ON"; bool hasJoin = false; - for (unsigned j = 0; j < AdjacencyList_[i].size(); ++ j) { - const unsigned connectionsJ = AdjacencyList_[AdjacencyList_[i][j].Target].size(); - - if (AdjacencyList_[i].size() > connectionsJ) { - continue; - } - - if (AdjacencyList_[i].size() == connectionsJ && i < AdjacencyList_[i][j].Target) { - continue; - } - + auto addJoinCodition = [&](int j) { if (hasJoin) { currentJoin += " AND"; } @@ -198,15 +188,24 @@ std::string TRelationGraph::MakeQuery() const { currentJoin += " " + getRelationName(i, AdjacencyList_[i][j].ColumnLHS) + " = " + getRelationName(AdjacencyList_[i][j].Target, AdjacencyList_[i][j].ColumnRHS); + + }; + + for (unsigned j = 0; j < AdjacencyList_[i].size(); ++ j) { + if (i < AdjacencyList_[i][j].Target) { + continue; + } + + addJoinCodition(j); } currentJoin += "\n"; if (hasJoin) { joinClause += currentJoin; - continue; - } else { - assert(fromClause.empty()); + } else if (fromClause.empty()) { fromClause = "SELECT *\nFROM " + getTableName(i) + "\n"; + } else { + joinClause += "CROSS JOIN " + getTableName(i) + "\n"; } } From fb6c1922919f071e6073b8cce2531afba4aee059 Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Fri, 24 Oct 2025 09:05:43 +0000 Subject: [PATCH 05/43] Implement vertex connectivity ordering for TRelationGraph to avoid cross joins --- .../ut/join/kqp_join_topology_generator.cpp | 55 +++++++++++++++++++ .../kqp/ut/join/kqp_join_topology_generator.h | 11 ++++ 2 files changed, 66 insertions(+) diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp index f167eee4e546..55851f66d06a 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp @@ -123,6 +123,16 @@ std::string TSchema::MakeDropQuery() const { return query; } +void TSchema::Rename(std::vector oldToNew) { + std::vector newTables(Tables_.size()); + + for (unsigned i = 0; i < oldToNew.size(); ++i) { + newTables[oldToNew[i]] = Tables_[i]; + } + + Tables_ = newTables; +} + TRelationGraph TRelationGraph::FromPrufer(std::mt19937 &mt, const std::vector& prufer, double newColumnProbability) { unsigned n = prufer.size() + 2; @@ -236,6 +246,51 @@ void TRelationGraph::DumpGraph(std::ostream &OS) const { OS << "}\n"; } +void TRelationGraph::ReorderDFS() { + std::vector visited(GetN(), false); + std::vector newOrder; + newOrder.reserve(GetN()); + + auto searchDepthFirst = [&](auto &&self, unsigned node) { + if (visited[node]) { + return; + } + + visited[node] = true; + newOrder.push_back(node); + + for (TEdge edge : AdjacencyList_[node]) { + self(self, edge.Target); + } + }; + + for (unsigned i = 0; i < GetN(); i++) { + searchDepthFirst(searchDepthFirst, i); + } + + std::vector oldToNew(GetN()); + for (unsigned i = 0; i < GetN(); i++) { + oldToNew[newOrder[i]] = i; + } + + Rename(oldToNew); + Schema_.Rename(oldToNew); +} + +void TRelationGraph::Rename(const std::vector &oldToNew) { + TAdjacencyList newGraph(GetN()); + for (unsigned u = 0; u < GetN(); ++ u) { + for (TEdge edge : AdjacencyList_[u]) { + unsigned v = edge.Target; + + edge.Target = oldToNew[v]; + newGraph[oldToNew[u]].push_back(edge); + } + } + + AdjacencyList_ = newGraph; +} + TSchemaStats TSchemaStats::MakeRandom(std::mt19937 &mt, const TSchema &schema, unsigned a, unsigned b) { std::uniform_int_distribution<> distribution(a, b); diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_generator.h b/ydb/core/kqp/ut/join/kqp_join_topology_generator.h index f34bb34deb35..36b87bdc12d7 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_generator.h +++ b/ydb/core/kqp/ut/join/kqp_join_topology_generator.h @@ -57,6 +57,8 @@ class TSchema { return Tables_.size(); } + void Rename(std::vector oldToNew); + private: std::vector Tables_; }; @@ -82,6 +84,15 @@ class TRelationGraph { return Schema_; } + unsigned GetN() const { + return AdjacencyList_.size(); + } + + void ReorderDFS(); + + void Rename(const std::vector &oldToNew); + + private: struct TEdge { unsigned Target; From 78d976f7fd9195b86983324797a550775acd4816 Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Fri, 24 Oct 2025 09:10:35 +0000 Subject: [PATCH 06/43] Migrate DumpGraph to IOutputStream --- ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp | 2 +- ydb/core/kqp/ut/join/kqp_join_topology_generator.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp index 55851f66d06a..184e656b808a 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp @@ -228,7 +228,7 @@ std::string TRelationGraph::MakeQuery() const { return std::move(fromClause) + std::move(joinClause) + ";\n"; } -void TRelationGraph::DumpGraph(std::ostream &OS) const { +void TRelationGraph::DumpGraph(IOutputStream &OS) const { OS << "graph {\n"; for (unsigned i = 0; i < AdjacencyList_.size(); ++ i) { for (unsigned j = 0; j < AdjacencyList_[i].size(); ++ j) { diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_generator.h b/ydb/core/kqp/ut/join/kqp_join_topology_generator.h index 36b87bdc12d7..9e41ff79404e 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_generator.h +++ b/ydb/core/kqp/ut/join/kqp_join_topology_generator.h @@ -78,7 +78,7 @@ class TRelationGraph { std::string MakeQuery() const; - void DumpGraph(std::ostream &OS) const; + void DumpGraph(IOutputStream &OS) const; const TSchema& GetSchema() const { return Schema_; From d4a2c49f56d46a2a5048229d9fc41e21c716c56c Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Fri, 24 Oct 2025 09:11:42 +0000 Subject: [PATCH 07/43] Fix assert in GenerateRandomPruferSequence --- ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp index 184e656b808a..c0a48ba797af 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp @@ -367,7 +367,7 @@ TRelationGraph GenerateFullyConnected(std::mt19937 &mt, unsigned numNodes, } static std::vector GenerateRandomPruferSequence(std::mt19937 &mt, unsigned numNodes) { - assert(n >= 2); + assert(numNodes >= 2); std::uniform_int_distribution<> distribution(0, numNodes - 1); std::vector prufer(numNodes - 2); From 8d78d1ff3706caf2adbdf7f9443dca514f30e726 Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Fri, 24 Oct 2025 09:13:05 +0000 Subject: [PATCH 08/43] Implement significantly more robust benchmarking method for join order --- ydb/core/kqp/ut/common/kqp_benches.h | 336 +++++++++++++++++++++ ydb/core/kqp/ut/join/kqp_join_order_ut.cpp | 226 +++++++++----- 2 files changed, 489 insertions(+), 73 deletions(-) create mode 100644 ydb/core/kqp/ut/common/kqp_benches.h diff --git a/ydb/core/kqp/ut/common/kqp_benches.h b/ydb/core/kqp/ut/common/kqp_benches.h new file mode 100644 index 000000000000..e5e5e71e1d67 --- /dev/null +++ b/ydb/core/kqp/ut/common/kqp_benches.h @@ -0,0 +1,336 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include + + +namespace NKikimr::NKqp { + + +template +ui64 MeasureTimeNanos(Lambda &&lambda) { + namespace Nsc = std::chrono; + using TClock = Nsc::high_resolution_clock; + + Nsc::time_point start = TClock::now(); + lambda(); + return Nsc::duration_cast(TClock::now() - start).count(); +} + +class TimeFormatter { +public: + static std::string Format(uint64_t valueNs, uint64_t uncertaintyNs) { + auto [unit, scale] = SelectUnit(valueNs); + + double scaledValue = valueNs / scale; + double scaledUncertainty = uncertaintyNs / scale; + + int decimalPlaces = GetDecimalPlaces(scaledUncertainty); + + std::ostringstream oss; + oss << std::fixed << std::setprecision(decimalPlaces); + oss << scaledValue << " " << unit << " ± " << scaledUncertainty << " " << unit; + + return oss.str(); + } + + static std::string Format(uint64_t valueNs) { + auto [unit, scale] = SelectUnit(valueNs); + double scaledValue = valueNs / scale; + + int decimalPlaces = GetDecimalPlacesForValue(scaledValue); + + std::ostringstream oss; + oss << std::fixed << std::setprecision(decimalPlaces); + oss << scaledValue << " " << unit; + + return oss.str(); + } + +private: + static std::pair SelectUnit(uint64_t nanoseconds) { + if (nanoseconds >= 1'000'000'000) { + return {"s", 1e9}; + } else if (nanoseconds >= 1'000'000) { + return {"ms", 1e6}; + } else if (nanoseconds >= 1'000) { + return {"μs", 1e3}; + } else { + return {"ns", 1.0}; + } + } + + static int GetDecimalPlaces(double scaledUncertainty) { + if (scaledUncertainty < 0.01) return 2; + + double log_val = std::log10(scaledUncertainty); + int magnitude = static_cast(std::floor(log_val)); + + if (scaledUncertainty >= 100.0) { + return 0; + } else if (scaledUncertainty >= 10.0) { + return 1; + } else if (scaledUncertainty >= 1.0) { + return 1; + } else { + return std::max(0, -magnitude + 1); + } + } + + static int GetDecimalPlacesForValue(double scaledValue) { + if (scaledValue < 0.01) return 4; + + double absValue = std::abs(scaledValue); + + if (absValue >= 1000.0) { + return 0; + } else if (absValue >= 100.0) { + return 1; + } else if (absValue >= 10.0) { + return 2; + } else if (absValue >= 1.0) { + return 2; + } else if (absValue >= 0.1) { + return 3; + } else { + return 4; + } + } + +}; + +template +struct TStatistics { + TValue N; + + TValue Min, Max; + + double Median; + double MAD; + + double Mean; + double Stdev; + + void Dump(IOutputStream &OS) const { + OS << "Median = " << TimeFormatter::Format(Median, MAD) << " (MAD)\n"; + OS << "Mean = " << TimeFormatter::Format(Mean, Stdev) << " (Std. dev.)\n"; + OS << "N = " << N << " [ " << TimeFormatter::Format(Min) << ", " << TimeFormatter::Format(Max) << " ]\n"; + } +}; + +// TValue is arithmetic and can be compared with itself, added to itself, and dividable by floats +// TValue is assumed to be cheap to pass by value +template +class TRunningStatistics { +public: + TRunningStatistics() + : MaxHeap_() + , MinHeap_() + , N_(0) + , Total_(0) + , Min_(std::nullopt) + , Max_(std::nullopt) + { + } + + void AddValue(TValue num) { + Total_ += num; + ++ N_; + + if (!Min_) { + Min_ = num; + } else { + Min_ = Min(*Min_, num); + } + + if (!Max_) { + Max_ = num; + } else { + Max_ = Max(*Max_, num); + }; + + if (MaxHeap_.empty() || num <= MaxHeap_[0]) { + MaxHeap_.push_back(num); + std::push_heap(MaxHeap_.begin(), MaxHeap_.end()); + } else { + MinHeap_.push_back(num); + std::push_heap(MinHeap_.begin(), MinHeap_.end(), std::greater{}); + } + + if (MaxHeap_.size() > MinHeap_.size() + 1) { + // MinHeap_.push(MaxHeap_.top()); + MinHeap_.push_back(MaxHeap_[0]); + std::push_heap(MinHeap_.begin(), MinHeap_.end(), std::greater{}); + + // MaxHeap_.pop(); + std::pop_heap(MaxHeap_.begin(), MaxHeap_.end()); + MaxHeap_.pop_back(); + } else if (MinHeap_.size() > MaxHeap_.size()) { + // MaxHeap_.push(MinHeap_.top()); + MaxHeap_.push_back(MinHeap_[0]); + std::push_heap(MaxHeap_.begin(), MaxHeap_.end()); + + // MinHeap_.pop(); + std::pop_heap(MinHeap_.begin(), MinHeap_.end(), std::greater{}); + MinHeap_.pop_back(); + } + } + + double GetMean() const { + assert(N_ != 0); + + return Total_ / static_cast(N_); + } + + double CalculateStdDev() const { + assert(N_ != 0); + + if (N_ == 1) { + return 0; + } + + auto mean = GetMean(); + + double deviationSquaresSum = 0; + auto addValue = [&](TValue value) { + double deviation = abs(value - mean); + deviationSquaresSum += deviation * deviation; + }; + + for (TValue value : MaxHeap_) { + addValue(value); + } + + for (TValue value : MinHeap_) { + addValue(value); + } + + return std::sqrt(deviationSquaresSum / (N_ - 1)); + } + + double GetMedian() const { + assert(!MaxHeap_.empty() || !MinHeap_.empty()); + + if (MaxHeap_.size() > MinHeap_.size()) { + return MaxHeap_[0]; + } else { + return (MaxHeap_[0] + MinHeap_[0]) / 2.0; + } + } + + double CalculateMAD() const { + assert(!MaxHeap_.empty() || !MinHeap_.empty()); + + std::vector deviation; + double median = GetMedian(); + + for (TValue value : MaxHeap_) { + deviation.push_back(std::abs(value - median)); + } + + for (TValue value : MinHeap_) { + deviation.push_back(std::abs(value - median)); + } + + std::sort(deviation.begin(), deviation.end()); + return (deviation[deviation.size() / 2] + deviation[(deviation.size() - 1) / 2]) / 2.0; + + } + + TValue GetMin() const { + assert(Min_); + return *Min_; + } + + TValue GetMax() const { + assert(Max_); + return *Max_; + } + + TValue GetTotal() const { + return Total_; + } + + ui32 GetN() const { + return N_; + } + + TStatistics GetStatistics() const { + return { + .N = GetN(), + .Min = GetMin(), + .Max = GetMax(), + .Median = GetMedian(), + .MAD = CalculateMAD(), + .Mean = GetMean(), + .Stdev = CalculateStdDev(), + }; + } + +private: + std::vector MaxHeap_; + std::vector MinHeap_; + + // std::priority_queue> MaxHeap_; + // std::priority_queue, std::greater> MinHeap_; + + ui32 N_; + TValue Total_; + + std::optional Min_; + std::optional Max_; +}; + +struct TRepeatedTestConfig { + ui32 MinRepeats; + ui32 MaxRepeats; + ui64 Timeout; +}; + +template +std::optional> RepeatedTest(TRepeatedTestConfig config, TLambda &&lambda) { + assert(config.MinRepeats >= 1); + + TRunningStatistics stats; + do { + bool hasTimedOut = false; + ui64 ellapsedTime = MeasureTimeNanos([&]() { hasTimedOut = lambda(); }); + + if (!hasTimedOut) { + return std::nullopt; + } + + stats.AddValue(ellapsedTime); + } while (stats.GetN() < config.MaxRepeats && + (stats.GetN() < config.MinRepeats || + stats.GetTotal() + stats.GetMedian() <= config.Timeout)); + + return stats.GetStatistics(); +} + + +struct TBenchmarkConfig { + TRepeatedTestConfig Warmup; + TRepeatedTestConfig Bench; +}; + +template +std::optional> Benchmark(TBenchmarkConfig config, Lambda &&lambda) { + if (config.Warmup.MaxRepeats != 0) { + auto warmupStats = RepeatedTest(config.Warmup, lambda); + if (!warmupStats) { + return std::nullopt; + } + } + + return RepeatedTest(config.Bench, lambda); +} + +} // end namespace NKikimr::NKqp diff --git a/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp index f4620e31b7ee..f63befedc00a 100644 --- a/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -618,15 +619,17 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) { } } - TString ExplainQuery(NYdb::NQuery::TSession session, const std::string &query) { + std::optional ExplainQuery(NYdb::NQuery::TSession session, const std::string &query) { auto explainRes = session.ExecuteQuery(query, NYdb::NQuery::TTxControl::NoTx(), NYdb::NQuery::TExecuteQuerySettings().ExecMode(NQuery::EExecMode::Explain) ).ExtractValueSync(); - Cout << query << "\n"; + if (explainRes.GetStatus() == EStatus::TIMEOUT) { + return std::nullopt; + } + explainRes.GetIssues().PrintTo(Cout); - Cout << "Status: " << explainRes.IsSuccess() << "\n"; UNIT_ASSERT_VALUES_EQUAL(explainRes.GetStatus(), EStatus::SUCCESS); return *explainRes.GetStats()->GetPlan(); @@ -635,9 +638,7 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) { std::vector ExecuteQuery(NYdb::NQuery::TSession session, std::string query) { auto execRes = session.ExecuteQuery(query, NYdb::NQuery::TTxControl::NoTx()).ExtractValueSync(); - Cout << query << "\n"; execRes.GetIssues().PrintTo(Cout); - Cout << "Status: " << execRes.IsSuccess() << "\n"; UNIT_ASSERT(execRes.IsSuccess()); return execRes.GetResultSets(); @@ -664,97 +665,122 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) { return queryWithShuffleElimination; } - struct TShuffleEliminationBenchResult { - unsigned TimeWithShuffleEliminationNanos = 0; - unsigned TimeWithoutShuffleEliminationNanos = 0; - }; + std::optional> BenchmarkExplain(TBenchmarkConfig config, NYdb::NQuery::TSession session, const TString& query) { + std::optional savedPlan = std::nullopt; + auto stats = Benchmark(config, [&]() -> bool { + auto plan = ExplainQuery(session, query); + if (!savedPlan) { + savedPlan = plan; + } - TShuffleEliminationBenchResult BenchmarkShuffleElimination(NYdb::NQuery::TSession session, const TString& query) { - namespace Nsc = std::chrono; - using TClock = Nsc::high_resolution_clock; + return !!plan; + }); - TString queryWithoutCBO = ConfigureQuery(query, /*enableShuffleElimination=*/false, /*optLevel=*/1); - unsigned nanosWithoutCBO = 0; - { - std::chrono::time_point start = TClock::now(); - ExplainQuery(session, queryWithoutCBO); - nanosWithoutCBO = Nsc::duration_cast(TClock::now() - start).count(); - Cout << "Nanoseconds for query without CBO: " << nanosWithoutCBO << "\n"; + if (!stats) { + Cout << "-------------------------------- TIMED OUT -------------------------------\n"; + return std::nullopt; } - TString queryWithShuffleElimination = ConfigureQuery(query, /*enableShuffleElimination=*/true); - TString planWithShuffleElimnation; - unsigned nanosWith = 0; - { + assert(savedPlan); + JustPrintPlan(*savedPlan); + Cout << "--------------------------------------------------------------------------\n"; - std::chrono::time_point start = TClock::now(); - planWithShuffleElimnation = ExplainQuery(session, queryWithShuffleElimination); - nanosWith = Nsc::duration_cast(TClock::now() - start).count(); - Cout << "Nanoseconds for CBO with shuffle elimination: " << nanosWith - nanosWithoutCBO << "\n"; - } + stats->Dump(Cout); - TString queryWithoutShuffleElimination = ConfigureQuery(query, /*enableShuffleElimination=*/false); - TString planWithoutShuffleElimnation; - unsigned nanosWithout = 0; - { - std::chrono::time_point start = TClock::now(); - planWithoutShuffleElimnation = ExplainQuery(session, queryWithoutShuffleElimination); - nanosWithout = Nsc::duration_cast(TClock::now() - start).count(); - Cout << "Nanoseconds for CBO without shuffle elimination: " << nanosWithout - nanosWithoutCBO << "\n"; - } + return stats; + } - JustPrintPlan(planWithShuffleElimnation); - JustPrintPlan(planWithoutShuffleElimnation); - return { nanosWith, nanosWithout }; + std::optional> BenchmarkShuffleElimination(TBenchmarkConfig config, NYdb::NQuery::TSession session, const TString& query) { + Cout << "--------------------------------- W/O CBO --------------------------------\n"; + BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/false, /*optLevel=*/1)); + Cout << "--------------------------------- CBO-SE ---------------------------------\n"; + BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/false, /*optLevel=*/2)); + Cout << "--------------------------------- CBO+SE ---------------------------------\n"; + auto result = BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/true, /*optLevel=*/2)); + Cout << "--------------------------------------------------------------------------\n"; + + return result; } - struct TTopologyShuffleEliminationResult { - double NewColumnProbability = 0.0; - int NumTables = 0; - TShuffleEliminationBenchResult Bench = {}; + TKikimrRunner GetCBOTestsYDB(TString stats, TDuration compilationTimeout) { + TVector settings; - void Dump(IOutputStream &OS) const { - OS << NewColumnProbability << " " << NumTables << " " - << Bench.TimeWithShuffleEliminationNanos << " " - << Bench.TimeWithoutShuffleEliminationNanos << "\n"; - } - }; + assert(!stats.empty()); + + NKikimrKqp::TKqpSetting setting; + setting.SetName("OptOverrideStatistics"); + setting.SetValue(stats); + settings.push_back(setting); + + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableConstantFolding(true); + appConfig.MutableTableServiceConfig()->SetEnableOrderOptimizaionFSM(true); + appConfig.MutableTableServiceConfig()->SetCompileTimeoutMs(compilationTimeout.MilliSeconds()); + + TKikimrSettings serverSettings(appConfig); + serverSettings.SetKqpSettings(settings); + + return TKikimrRunner(serverSettings); + } template void BenchmarkShuffleEliminationOnTopology(int maxNumTables, double probabilityStep, unsigned seed = 0) { + auto config = TBenchmarkConfig { + .Warmup = { + .MinRepeats = 1, + .MaxRepeats = 3, + .Timeout = 100'000'000, + }, + + .Bench = { + .MinRepeats = 5, + .MaxRepeats = 100, + .Timeout = 10'000'000'000, + }, + }; + std::mt19937 mt(seed); TSchema fullSchema = TSchema::MakeWithEnoughColumns(maxNumTables); TString stats = TSchemaStats::MakeRandom(mt, fullSchema, 7, 10).ToJSON(); - auto kikimr = GetKikimrWithJoinSettings(/*useStreamLookupJoin=*/false, /*stats=*/stats, /*useCBO=*/true, TExecuteParams{}); + auto kikimr = GetCBOTestsYDB(stats, TDuration::Seconds(3)); auto db = kikimr.GetQueryClient(); auto session = db.GetSession().GetValueSync().GetSession(); - std::vector data; - int totalCount = 0; - for (double probability = 0; probability <= 1.0; probability += probabilityStep) { - for (int i = 2 /* one table is not possible with all topoloties, so 2 */; i < maxNumTables; i ++) { - TRelationGraph graph = Lambda(mt, i, 0.5); - - ExecuteQuery(session, graph.GetSchema().MakeCreateQuery()); - - auto resultTime = BenchmarkShuffleElimination(session, graph.MakeQuery()); - data.push_back({probability, i, resultTime}); - - Cout << "totalCount: " << ++ totalCount << "\n"; - Cout << "result: "; - data.back().Dump(Cout); - - ExecuteQuery(session, graph.GetSchema().MakeDropQuery()); + for (int i = 2 /* one table is not possible with all topoloties, so 2 */; i < maxNumTables; i ++) { + for (double probability = 0; probability <= 1.0; probability += probabilityStep) { + TRelationGraph graph = Lambda(mt, i, probability); + + Cout << "\n\n"; + Cout << "================================= CREATE =================================\n"; + graph.DumpGraph(Cout); + + Cout << "================================= REORDER ================================\n"; + graph.ReorderDFS(); + graph.DumpGraph(Cout); + + Cout << "================================= PREPARE ================================\n"; + auto creationQuery = graph.GetSchema().MakeCreateQuery(); + Cout << creationQuery; + ExecuteQuery(session, creationQuery); + + Cout << "================================= BENCHMARK ==============================\n"; + TString query = graph.MakeQuery(); + Cout << query; + auto resultTime = BenchmarkShuffleElimination(config, session, query); + + Cout << "================================= FINALIZE ===============================\n"; + auto deletionQuery = graph.GetSchema().MakeDropQuery(); + Cout << deletionQuery; + ExecuteQuery(session, deletionQuery); + Cout << "==========================================================================\n"; + if (!resultTime) { + continue; + } } } - - // Printing a report of all measurements - for (const auto &result: data) { - result.Dump(Cout); - } } void CheckJoinCardinality(const TString& queryPath, const TString& statsPath, const TString& joinKind, double card, bool useStreamLookupJoin, bool useColumnStore) { @@ -1020,7 +1046,61 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) { } Y_UNIT_TEST(ShuffleEliminationBenchmarkOnRandomTree) { - BenchmarkShuffleEliminationOnTopology(/*maxNumTables=*/15, /*probabilityStep=*/0.2); + BenchmarkShuffleEliminationOnTopology(/*maxNumTables=*/25, /*probabilityStep=*/0.2); + } + + Y_UNIT_TEST(TRunningStatistics) { + TRunningStatistics stats; + + stats.AddValue(1); + UNIT_ASSERT_EQUAL(stats.GetMedian(), 1); + UNIT_ASSERT_EQUAL(stats.GetMean(), 1 / 1.0); + UNIT_ASSERT_EQUAL(stats.GetN(), 1); + + stats.AddValue(1); + UNIT_ASSERT_EQUAL(stats.GetMedian(), (1 + 1) / 2.0); + UNIT_ASSERT_EQUAL(stats.GetMean(), 2 / 2.0); + UNIT_ASSERT_EQUAL(stats.GetN(), 2); + + stats.AddValue(1); + UNIT_ASSERT_EQUAL(stats.GetMedian(), 1); + UNIT_ASSERT_EQUAL(stats.GetMean(), 3 / 3.0); + UNIT_ASSERT_EQUAL(stats.GetN(), 3); + + stats.AddValue(1); + UNIT_ASSERT_EQUAL(stats.GetMedian(), (1 + 1) / 2.0); + UNIT_ASSERT_EQUAL(stats.GetMean(), 4 / 4.0); + UNIT_ASSERT_EQUAL(stats.GetN(), 4); + + stats.AddValue(3); + UNIT_ASSERT_EQUAL(stats.GetMedian(), 1); + UNIT_ASSERT_EQUAL(stats.GetMean(), 7 / 5.0); + UNIT_ASSERT_EQUAL(stats.GetN(), 5); + + stats.AddValue(5); + UNIT_ASSERT_EQUAL(stats.GetMedian(), (1 + 1) / 2.0); + UNIT_ASSERT_EQUAL(stats.GetMean(), 12 / 6.0); + UNIT_ASSERT_EQUAL(stats.GetN(), 6); + + stats.AddValue(7); + UNIT_ASSERT_EQUAL(stats.GetMedian(), 1); + UNIT_ASSERT_EQUAL(stats.GetMean(), 19 / 7.0); + UNIT_ASSERT_EQUAL(stats.GetN(), 7); + + stats.AddValue(7); + UNIT_ASSERT_EQUAL(stats.GetMedian(), (1 + 3) / 2.0); + UNIT_ASSERT_EQUAL(stats.GetMean(), 26 / 8.0); + UNIT_ASSERT_EQUAL(stats.GetN(), 8); + + stats.AddValue(8); + UNIT_ASSERT_EQUAL(stats.GetMedian(), 3); + UNIT_ASSERT_EQUAL(stats.GetMean(), 34 / 9.0); + UNIT_ASSERT_EQUAL(stats.GetN(), 9); + + stats.AddValue(100); + UNIT_ASSERT_EQUAL(stats.GetMedian(), (3 + 5) / 2.0); + UNIT_ASSERT_EQUAL(stats.GetMean(), 134 / 10.0); + UNIT_ASSERT_EQUAL(stats.GetN(), 10); } Y_UNIT_TEST_TWIN(ShuffleEliminationOneJoin, EnableSeparationComputeActorsFromRead) { From 5e2d33b590de3c8fca9c97642bf6a566ad691c73 Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Fri, 24 Oct 2025 09:15:50 +0000 Subject: [PATCH 09/43] Implement degree sequence calculation for TRelationGraph --- ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp | 10 ++++++++++ ydb/core/kqp/ut/join/kqp_join_topology_generator.h | 2 ++ 2 files changed, 12 insertions(+) diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp index c0a48ba797af..0ba8841c896f 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp @@ -246,6 +246,16 @@ void TRelationGraph::DumpGraph(IOutputStream &OS) const { OS << "}\n"; } +std::vector TRelationGraph::GetDegrees() const { + std::vector degrees(AdjacencyList_.size()); + for (unsigned i = 0; i < AdjacencyList_.size(); ++ i) { + degrees[i] = AdjacencyList_[i].size(); + } + + std::sort(degrees.begin(), degrees.end()); + return degrees; +} + void TRelationGraph::ReorderDFS() { std::vector visited(GetN(), false); std::vector newOrder; diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_generator.h b/ydb/core/kqp/ut/join/kqp_join_topology_generator.h index 9e41ff79404e..2d5f23086aaf 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_generator.h +++ b/ydb/core/kqp/ut/join/kqp_join_topology_generator.h @@ -84,6 +84,8 @@ class TRelationGraph { return Schema_; } + std::vector GetDegrees() const; + unsigned GetN() const { return AdjacencyList_.size(); } From 911a9dee58fd1e2e8ffbb4ca37e07ee44d58ebd8 Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Fri, 24 Oct 2025 09:17:45 +0000 Subject: [PATCH 10/43] Refactor kqp_join_topology_generator --- ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp | 6 ++---- ydb/core/kqp/ut/join/kqp_join_topology_generator.h | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp index 0ba8841c896f..912098063384 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp @@ -1,11 +1,9 @@ #include "kqp_join_topology_generator.h" #include -#include #include #include #include -#include namespace NKikimr::NKqp { @@ -185,7 +183,7 @@ std::string TRelationGraph::MakeQuery() const { std::string joinClause; for (unsigned i = 0; i < AdjacencyList_.size(); ++ i) { std::string currentJoin = - "JOIN " + TLexicographicalNameGenerator::getName(i, /*lowerCase=*/false) + " ON"; + "JOIN " + getTableName(i) + " ON"; bool hasJoin = false; auto addJoinCodition = [&](int j) { @@ -323,7 +321,7 @@ std::string TSchemaStats::ToJSON() const { if (i != 0) ss << ","; - ss << "\"/Root/" << TLexicographicalNameGenerator::getName(i, /*lowerCase=*/false) << "\": "; + ss << "\"" << getTablePath(i) << "\": "; ss << "{"; ss << "\"" << "n_rows" << "\": " << Stats_[i].RowSize << ", "; ss << "\"" << "byte_size" << "\": " << Stats_[i].ByteSize; diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_generator.h b/ydb/core/kqp/ut/join/kqp_join_topology_generator.h index 2d5f23086aaf..25a5a04cbd8e 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_generator.h +++ b/ydb/core/kqp/ut/join/kqp_join_topology_generator.h @@ -1,9 +1,9 @@ #pragma once +#include + #include -#include #include -#include #include #include From 865ba769f2ffafe95157150d673c6777e751da70 Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Mon, 27 Oct 2025 03:17:02 +0000 Subject: [PATCH 11/43] Disable sample tables in join topology tests --- ydb/core/kqp/ut/join/kqp_join_order_ut.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp index f63befedc00a..9155053ee647 100644 --- a/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp @@ -719,6 +719,7 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) { appConfig.MutableTableServiceConfig()->SetCompileTimeoutMs(compilationTimeout.MilliSeconds()); TKikimrSettings serverSettings(appConfig); + serverSettings.SetWithSampleTables(false); serverSettings.SetKqpSettings(settings); return TKikimrRunner(serverSettings); From 2775cd715f1c24b047d49361b328cdea905a4eb9 Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Mon, 27 Oct 2025 03:22:10 +0000 Subject: [PATCH 12/43] Improve timeout handling in join topology tests --- ydb/core/kqp/ut/common/kqp_benches.h | 4 +- ydb/core/kqp/ut/join/kqp_join_order_ut.cpp | 86 +++++++++++++--------- 2 files changed, 52 insertions(+), 38 deletions(-) diff --git a/ydb/core/kqp/ut/common/kqp_benches.h b/ydb/core/kqp/ut/common/kqp_benches.h index e5e5e71e1d67..8da40fe8d068 100644 --- a/ydb/core/kqp/ut/common/kqp_benches.h +++ b/ydb/core/kqp/ut/common/kqp_benches.h @@ -301,9 +301,9 @@ std::optional> RepeatedTest(TRepeatedTestConfig config, TLambd TRunningStatistics stats; do { bool hasTimedOut = false; - ui64 ellapsedTime = MeasureTimeNanos([&]() { hasTimedOut = lambda(); }); + ui64 ellapsedTime = MeasureTimeNanos([&]() { hasTimedOut = !lambda(); }); - if (!hasTimedOut) { + if (hasTimedOut) { return std::nullopt; } diff --git a/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp index 9155053ee647..eccbd9df0e09 100644 --- a/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp @@ -635,13 +635,17 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) { return *explainRes.GetStats()->GetPlan(); } - std::vector ExecuteQuery(NYdb::NQuery::TSession session, std::string query) { + bool ExecuteQuery(NYdb::NQuery::TSession session, std::string query) { auto execRes = session.ExecuteQuery(query, NYdb::NQuery::TTxControl::NoTx()).ExtractValueSync(); + if (execRes.GetStatus() == EStatus::TIMEOUT) { + return false; + } + execRes.GetIssues().PrintTo(Cout); UNIT_ASSERT(execRes.IsSuccess()); - return execRes.GetResultSets(); + return true; } void JustPrintPlan(const TString &plan) { @@ -725,8 +729,44 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) { return TKikimrRunner(serverSettings); } + bool BenchmarkShuffleEliminationOnTopology(TBenchmarkConfig config, NYdb::NQuery::TSession session, TRelationGraph graph) { + Cout << "\n\n"; + Cout << "================================= CREATE =================================\n"; + graph.DumpGraph(Cout); + + Cout << "================================= REORDER ================================\n"; + graph.ReorderDFS(); + graph.DumpGraph(Cout); + + Cout << "================================= PREPARE ================================\n"; + auto creationQuery = graph.GetSchema().MakeCreateQuery(); + Cout << creationQuery; + if (!ExecuteQuery(session, creationQuery)) { + return false; + } + + Cout << "================================= BENCHMARK ==============================\n"; + TString query = graph.MakeQuery(); + Cout << query; + auto resultTime = BenchmarkShuffleElimination(config, session, query); + + Cout << "================================= FINALIZE ===============================\n"; + auto deletionQuery = graph.GetSchema().MakeDropQuery(); + Cout << deletionQuery; + if (!ExecuteQuery(session, deletionQuery)) { // TODO: this is really bad probably? + return false; + } + Cout << "==========================================================================\n"; + + if (!resultTime) { + return false; // query timed out + } + + return true; + } + template - void BenchmarkShuffleEliminationOnTopology(int maxNumTables, double probabilityStep, unsigned seed = 0) { + void BenchmarkShuffleEliminationOnTopologies(int maxNumTables, double probabilityStep, unsigned seed = 0) { auto config = TBenchmarkConfig { .Warmup = { .MinRepeats = 1, @@ -746,40 +786,14 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) { TSchema fullSchema = TSchema::MakeWithEnoughColumns(maxNumTables); TString stats = TSchemaStats::MakeRandom(mt, fullSchema, 7, 10).ToJSON(); - auto kikimr = GetCBOTestsYDB(stats, TDuration::Seconds(3)); + auto kikimr = GetCBOTestsYDB(stats, TDuration::Seconds(10)); auto db = kikimr.GetQueryClient(); auto session = db.GetSession().GetValueSync().GetSession(); - for (int i = 2 /* one table is not possible with all topoloties, so 2 */; i < maxNumTables; i ++) { + for (int i = 2 /* one table is not possible with all topologies, so 2 */; i < maxNumTables; i ++) { for (double probability = 0; probability <= 1.0; probability += probabilityStep) { TRelationGraph graph = Lambda(mt, i, probability); - - Cout << "\n\n"; - Cout << "================================= CREATE =================================\n"; - graph.DumpGraph(Cout); - - Cout << "================================= REORDER ================================\n"; - graph.ReorderDFS(); - graph.DumpGraph(Cout); - - Cout << "================================= PREPARE ================================\n"; - auto creationQuery = graph.GetSchema().MakeCreateQuery(); - Cout << creationQuery; - ExecuteQuery(session, creationQuery); - - Cout << "================================= BENCHMARK ==============================\n"; - TString query = graph.MakeQuery(); - Cout << query; - auto resultTime = BenchmarkShuffleElimination(config, session, query); - - Cout << "================================= FINALIZE ===============================\n"; - auto deletionQuery = graph.GetSchema().MakeDropQuery(); - Cout << deletionQuery; - ExecuteQuery(session, deletionQuery); - Cout << "==========================================================================\n"; - if (!resultTime) { - continue; - } + BenchmarkShuffleEliminationOnTopology(config, session, graph); } } } @@ -1035,19 +1049,19 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) { } Y_UNIT_TEST(ShuffleEliminationBenchmarkOnStar) { - BenchmarkShuffleEliminationOnTopology(/*maxNumTables=*/15, /*probabilityStep=*/0.2); + BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/15, /*probabilityStep=*/0.2); } Y_UNIT_TEST(ShuffleEliminationBenchmarkOnLine) { - BenchmarkShuffleEliminationOnTopology(/*maxNumTables=*/15, /*probabilityStep=*/0.2); + BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/15, /*probabilityStep=*/0.2); } Y_UNIT_TEST(ShuffleEliminationBenchmarkOnFullyConnected) { - BenchmarkShuffleEliminationOnTopology(/*maxNumTables=*/15, /*probabilityStep=*/0.2); + BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/15, /*probabilityStep=*/0.2); } Y_UNIT_TEST(ShuffleEliminationBenchmarkOnRandomTree) { - BenchmarkShuffleEliminationOnTopology(/*maxNumTables=*/25, /*probabilityStep=*/0.2); + BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/25, /*probabilityStep=*/0.2); } Y_UNIT_TEST(TRunningStatistics) { From 6341d973efe17e721dc61051e51c664c1c6731b2 Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Mon, 27 Oct 2025 03:42:53 +0000 Subject: [PATCH 13/43] Separate topology tests into separate file --- ydb/core/kqp/ut/join/kqp_join_order_ut.cpp | 252 ---------------- ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp | 271 ++++++++++++++++++ ydb/core/kqp/ut/join/ya.make | 1 + 3 files changed, 272 insertions(+), 252 deletions(-) create mode 100644 ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp diff --git a/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp index eccbd9df0e09..51d70af316a1 100644 --- a/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp @@ -1,5 +1,4 @@ #include -#include #include #include @@ -12,8 +11,6 @@ #include #include -#include "kqp_join_topology_generator.h" - namespace NKikimr { namespace NKqp { @@ -619,185 +616,6 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) { } } - std::optional ExplainQuery(NYdb::NQuery::TSession session, const std::string &query) { - auto explainRes = session.ExecuteQuery(query, - NYdb::NQuery::TTxControl::NoTx(), - NYdb::NQuery::TExecuteQuerySettings().ExecMode(NQuery::EExecMode::Explain) - ).ExtractValueSync(); - - if (explainRes.GetStatus() == EStatus::TIMEOUT) { - return std::nullopt; - } - - explainRes.GetIssues().PrintTo(Cout); - UNIT_ASSERT_VALUES_EQUAL(explainRes.GetStatus(), EStatus::SUCCESS); - - return *explainRes.GetStats()->GetPlan(); - } - - bool ExecuteQuery(NYdb::NQuery::TSession session, std::string query) { - auto execRes = session.ExecuteQuery(query, NYdb::NQuery::TTxControl::NoTx()).ExtractValueSync(); - - if (execRes.GetStatus() == EStatus::TIMEOUT) { - return false; - } - - execRes.GetIssues().PrintTo(Cout); - UNIT_ASSERT(execRes.IsSuccess()); - - return true; - } - - void JustPrintPlan(const TString &plan) { - NYdb::NConsoleClient::TQueryPlanPrinter queryPlanPrinter( - NYdb::NConsoleClient::EDataFormat::PrettyTable, - /*analyzeMode=*/true, Cout, /*maxWidth=*/0 - ); - - queryPlanPrinter.Print(plan); - } - - std::string ConfigureQuery(const std::string &query, bool enableShuffleElimination = false, unsigned optLevel = 2) { - std::string queryWithShuffleElimination = "PRAGMA ydb.OptShuffleElimination = \""; - queryWithShuffleElimination += enableShuffleElimination ? "true" : "false"; - queryWithShuffleElimination += "\";\n"; - queryWithShuffleElimination += "PRAGMA ydb.MaxDPHypDPTableSize='4294967295';\n"; - queryWithShuffleElimination += "PRAGMA ydb.ForceShuffleElimination='true';\n"; - queryWithShuffleElimination += "PRAGMA ydb.CostBasedOptimizationLevel = \"" + std::to_string(optLevel) + "\";\n"; - queryWithShuffleElimination += query; - - return queryWithShuffleElimination; - } - - std::optional> BenchmarkExplain(TBenchmarkConfig config, NYdb::NQuery::TSession session, const TString& query) { - std::optional savedPlan = std::nullopt; - auto stats = Benchmark(config, [&]() -> bool { - auto plan = ExplainQuery(session, query); - if (!savedPlan) { - savedPlan = plan; - } - - return !!plan; - }); - - if (!stats) { - Cout << "-------------------------------- TIMED OUT -------------------------------\n"; - return std::nullopt; - } - - assert(savedPlan); - JustPrintPlan(*savedPlan); - Cout << "--------------------------------------------------------------------------\n"; - - stats->Dump(Cout); - - return stats; - } - - - std::optional> BenchmarkShuffleElimination(TBenchmarkConfig config, NYdb::NQuery::TSession session, const TString& query) { - Cout << "--------------------------------- W/O CBO --------------------------------\n"; - BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/false, /*optLevel=*/1)); - Cout << "--------------------------------- CBO-SE ---------------------------------\n"; - BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/false, /*optLevel=*/2)); - Cout << "--------------------------------- CBO+SE ---------------------------------\n"; - auto result = BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/true, /*optLevel=*/2)); - Cout << "--------------------------------------------------------------------------\n"; - - return result; - } - - TKikimrRunner GetCBOTestsYDB(TString stats, TDuration compilationTimeout) { - TVector settings; - - assert(!stats.empty()); - - NKikimrKqp::TKqpSetting setting; - setting.SetName("OptOverrideStatistics"); - setting.SetValue(stats); - settings.push_back(setting); - - NKikimrConfig::TAppConfig appConfig; - appConfig.MutableTableServiceConfig()->SetEnableConstantFolding(true); - appConfig.MutableTableServiceConfig()->SetEnableOrderOptimizaionFSM(true); - appConfig.MutableTableServiceConfig()->SetCompileTimeoutMs(compilationTimeout.MilliSeconds()); - - TKikimrSettings serverSettings(appConfig); - serverSettings.SetWithSampleTables(false); - serverSettings.SetKqpSettings(settings); - - return TKikimrRunner(serverSettings); - } - - bool BenchmarkShuffleEliminationOnTopology(TBenchmarkConfig config, NYdb::NQuery::TSession session, TRelationGraph graph) { - Cout << "\n\n"; - Cout << "================================= CREATE =================================\n"; - graph.DumpGraph(Cout); - - Cout << "================================= REORDER ================================\n"; - graph.ReorderDFS(); - graph.DumpGraph(Cout); - - Cout << "================================= PREPARE ================================\n"; - auto creationQuery = graph.GetSchema().MakeCreateQuery(); - Cout << creationQuery; - if (!ExecuteQuery(session, creationQuery)) { - return false; - } - - Cout << "================================= BENCHMARK ==============================\n"; - TString query = graph.MakeQuery(); - Cout << query; - auto resultTime = BenchmarkShuffleElimination(config, session, query); - - Cout << "================================= FINALIZE ===============================\n"; - auto deletionQuery = graph.GetSchema().MakeDropQuery(); - Cout << deletionQuery; - if (!ExecuteQuery(session, deletionQuery)) { // TODO: this is really bad probably? - return false; - } - Cout << "==========================================================================\n"; - - if (!resultTime) { - return false; // query timed out - } - - return true; - } - - template - void BenchmarkShuffleEliminationOnTopologies(int maxNumTables, double probabilityStep, unsigned seed = 0) { - auto config = TBenchmarkConfig { - .Warmup = { - .MinRepeats = 1, - .MaxRepeats = 3, - .Timeout = 100'000'000, - }, - - .Bench = { - .MinRepeats = 5, - .MaxRepeats = 100, - .Timeout = 10'000'000'000, - }, - }; - - std::mt19937 mt(seed); - - TSchema fullSchema = TSchema::MakeWithEnoughColumns(maxNumTables); - TString stats = TSchemaStats::MakeRandom(mt, fullSchema, 7, 10).ToJSON(); - - auto kikimr = GetCBOTestsYDB(stats, TDuration::Seconds(10)); - auto db = kikimr.GetQueryClient(); - auto session = db.GetSession().GetValueSync().GetSession(); - - for (int i = 2 /* one table is not possible with all topologies, so 2 */; i < maxNumTables; i ++) { - for (double probability = 0; probability <= 1.0; probability += probabilityStep) { - TRelationGraph graph = Lambda(mt, i, probability); - BenchmarkShuffleEliminationOnTopology(config, session, graph); - } - } - } - void CheckJoinCardinality(const TString& queryPath, const TString& statsPath, const TString& joinKind, double card, bool useStreamLookupJoin, bool useColumnStore) { auto kikimr = GetKikimrWithJoinSettings(useStreamLookupJoin, GetStatic(statsPath), true); kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableViews(true); @@ -1048,76 +866,6 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) { UNIT_ASSERT_C(join.Join == "InnerJoin (Grace)", join.Join); } - Y_UNIT_TEST(ShuffleEliminationBenchmarkOnStar) { - BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/15, /*probabilityStep=*/0.2); - } - - Y_UNIT_TEST(ShuffleEliminationBenchmarkOnLine) { - BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/15, /*probabilityStep=*/0.2); - } - - Y_UNIT_TEST(ShuffleEliminationBenchmarkOnFullyConnected) { - BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/15, /*probabilityStep=*/0.2); - } - - Y_UNIT_TEST(ShuffleEliminationBenchmarkOnRandomTree) { - BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/25, /*probabilityStep=*/0.2); - } - - Y_UNIT_TEST(TRunningStatistics) { - TRunningStatistics stats; - - stats.AddValue(1); - UNIT_ASSERT_EQUAL(stats.GetMedian(), 1); - UNIT_ASSERT_EQUAL(stats.GetMean(), 1 / 1.0); - UNIT_ASSERT_EQUAL(stats.GetN(), 1); - - stats.AddValue(1); - UNIT_ASSERT_EQUAL(stats.GetMedian(), (1 + 1) / 2.0); - UNIT_ASSERT_EQUAL(stats.GetMean(), 2 / 2.0); - UNIT_ASSERT_EQUAL(stats.GetN(), 2); - - stats.AddValue(1); - UNIT_ASSERT_EQUAL(stats.GetMedian(), 1); - UNIT_ASSERT_EQUAL(stats.GetMean(), 3 / 3.0); - UNIT_ASSERT_EQUAL(stats.GetN(), 3); - - stats.AddValue(1); - UNIT_ASSERT_EQUAL(stats.GetMedian(), (1 + 1) / 2.0); - UNIT_ASSERT_EQUAL(stats.GetMean(), 4 / 4.0); - UNIT_ASSERT_EQUAL(stats.GetN(), 4); - - stats.AddValue(3); - UNIT_ASSERT_EQUAL(stats.GetMedian(), 1); - UNIT_ASSERT_EQUAL(stats.GetMean(), 7 / 5.0); - UNIT_ASSERT_EQUAL(stats.GetN(), 5); - - stats.AddValue(5); - UNIT_ASSERT_EQUAL(stats.GetMedian(), (1 + 1) / 2.0); - UNIT_ASSERT_EQUAL(stats.GetMean(), 12 / 6.0); - UNIT_ASSERT_EQUAL(stats.GetN(), 6); - - stats.AddValue(7); - UNIT_ASSERT_EQUAL(stats.GetMedian(), 1); - UNIT_ASSERT_EQUAL(stats.GetMean(), 19 / 7.0); - UNIT_ASSERT_EQUAL(stats.GetN(), 7); - - stats.AddValue(7); - UNIT_ASSERT_EQUAL(stats.GetMedian(), (1 + 3) / 2.0); - UNIT_ASSERT_EQUAL(stats.GetMean(), 26 / 8.0); - UNIT_ASSERT_EQUAL(stats.GetN(), 8); - - stats.AddValue(8); - UNIT_ASSERT_EQUAL(stats.GetMedian(), 3); - UNIT_ASSERT_EQUAL(stats.GetMean(), 34 / 9.0); - UNIT_ASSERT_EQUAL(stats.GetN(), 9); - - stats.AddValue(100); - UNIT_ASSERT_EQUAL(stats.GetMedian(), (3 + 5) / 2.0); - UNIT_ASSERT_EQUAL(stats.GetMean(), 134 / 10.0); - UNIT_ASSERT_EQUAL(stats.GetN(), 10); - } - Y_UNIT_TEST_TWIN(ShuffleEliminationOneJoin, EnableSeparationComputeActorsFromRead) { auto [plan, _] = ExecuteJoinOrderTestGenericQueryWithStats("queries/shuffle_elimination_one_join.sql", "stats/tpch1000s.json", false, true, true, {.EnableSeparationComputeActorsFromRead = EnableSeparationComputeActorsFromRead}); auto joinFinder = TFindJoinWithLabels(plan); diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp new file mode 100644 index 000000000000..41a958d59dd2 --- /dev/null +++ b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp @@ -0,0 +1,271 @@ +#include +#include + +#include +#include +#include + +#include "kqp_join_topology_generator.h" + +namespace NKikimr { +namespace NKqp { + +using namespace NYdb; +using namespace NYdb::NTable; + +Y_UNIT_TEST_SUITE(KqpJoinTopology) { + + std::optional ExplainQuery(NYdb::NQuery::TSession session, const std::string &query) { + auto explainRes = session.ExecuteQuery(query, + NYdb::NQuery::TTxControl::NoTx(), + NYdb::NQuery::TExecuteQuerySettings().ExecMode(NQuery::EExecMode::Explain) + ).ExtractValueSync(); + + if (explainRes.GetStatus() == EStatus::TIMEOUT) { + return std::nullopt; + } + + explainRes.GetIssues().PrintTo(Cout); + UNIT_ASSERT_VALUES_EQUAL(explainRes.GetStatus(), EStatus::SUCCESS); + + return *explainRes.GetStats()->GetPlan(); + } + + bool ExecuteQuery(NYdb::NQuery::TSession session, std::string query) { + auto execRes = session.ExecuteQuery(query, NYdb::NQuery::TTxControl::NoTx()).ExtractValueSync(); + + if (execRes.GetStatus() == EStatus::TIMEOUT) { + return false; + } + + execRes.GetIssues().PrintTo(Cout); + UNIT_ASSERT(execRes.IsSuccess()); + + return true; + } + + void JustPrintPlan(const TString &plan) { + NYdb::NConsoleClient::TQueryPlanPrinter queryPlanPrinter( + NYdb::NConsoleClient::EDataFormat::PrettyTable, + /*analyzeMode=*/true, Cout, /*maxWidth=*/0 + ); + + queryPlanPrinter.Print(plan); + } + + std::string ConfigureQuery(const std::string &query, bool enableShuffleElimination = false, unsigned optLevel = 2) { + std::string queryWithShuffleElimination = "PRAGMA ydb.OptShuffleElimination = \""; + queryWithShuffleElimination += enableShuffleElimination ? "true" : "false"; + queryWithShuffleElimination += "\";\n"; + queryWithShuffleElimination += "PRAGMA ydb.MaxDPHypDPTableSize='4294967295';\n"; + queryWithShuffleElimination += "PRAGMA ydb.ForceShuffleElimination='true';\n"; + queryWithShuffleElimination += "PRAGMA ydb.CostBasedOptimizationLevel = \"" + std::to_string(optLevel) + "\";\n"; + queryWithShuffleElimination += query; + + return queryWithShuffleElimination; + } + + std::optional> BenchmarkExplain(TBenchmarkConfig config, NYdb::NQuery::TSession session, const TString& query) { + std::optional savedPlan = std::nullopt; + auto stats = Benchmark(config, [&]() -> bool { + auto plan = ExplainQuery(session, query); + if (!savedPlan) { + savedPlan = plan; + } + + return !!plan; + }); + + if (!stats) { + Cout << "-------------------------------- TIMED OUT -------------------------------\n"; + return std::nullopt; + } + + assert(savedPlan); + JustPrintPlan(*savedPlan); + Cout << "--------------------------------------------------------------------------\n"; + + stats->Dump(Cout); + + return stats; + } + + + std::optional> BenchmarkShuffleElimination(TBenchmarkConfig config, NYdb::NQuery::TSession session, const TString& query) { + Cout << "--------------------------------- W/O CBO --------------------------------\n"; + BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/false, /*optLevel=*/1)); + Cout << "--------------------------------- CBO-SE ---------------------------------\n"; + BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/false, /*optLevel=*/2)); + Cout << "--------------------------------- CBO+SE ---------------------------------\n"; + auto result = BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/true, /*optLevel=*/2)); + Cout << "--------------------------------------------------------------------------\n"; + + return result; + } + + TKikimrRunner GetCBOTestsYDB(TString stats, TDuration compilationTimeout) { + TVector settings; + + assert(!stats.empty()); + + NKikimrKqp::TKqpSetting setting; + setting.SetName("OptOverrideStatistics"); + setting.SetValue(stats); + settings.push_back(setting); + + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableConstantFolding(true); + appConfig.MutableTableServiceConfig()->SetEnableOrderOptimizaionFSM(true); + appConfig.MutableTableServiceConfig()->SetCompileTimeoutMs(compilationTimeout.MilliSeconds()); + + TKikimrSettings serverSettings(appConfig); + serverSettings.SetWithSampleTables(false); + serverSettings.SetKqpSettings(settings); + + return TKikimrRunner(serverSettings); + } + + bool BenchmarkShuffleEliminationOnTopology(TBenchmarkConfig config, NYdb::NQuery::TSession session, TRelationGraph graph) { + Cout << "\n\n"; + Cout << "================================= CREATE =================================\n"; + graph.DumpGraph(Cout); + + Cout << "================================= REORDER ================================\n"; + graph.ReorderDFS(); + graph.DumpGraph(Cout); + + Cout << "================================= PREPARE ================================\n"; + auto creationQuery = graph.GetSchema().MakeCreateQuery(); + Cout << creationQuery; + if (!ExecuteQuery(session, creationQuery)) { + return false; + } + + Cout << "================================= BENCHMARK ==============================\n"; + TString query = graph.MakeQuery(); + Cout << query; + auto resultTime = BenchmarkShuffleElimination(config, session, query); + + Cout << "================================= FINALIZE ===============================\n"; + auto deletionQuery = graph.GetSchema().MakeDropQuery(); + Cout << deletionQuery; + if (!ExecuteQuery(session, deletionQuery)) { // TODO: this is really bad probably? + return false; + } + Cout << "==========================================================================\n"; + + if (!resultTime) { + return false; // query timed out + } + + return true; + } + + template + void BenchmarkShuffleEliminationOnTopologies(int maxNumTables, double probabilityStep, unsigned seed = 0) { + auto config = TBenchmarkConfig { + .Warmup = { + .MinRepeats = 1, + .MaxRepeats = 3, + .Timeout = 10'000'000, + }, + + .Bench = { + .MinRepeats = 3, + .MaxRepeats = 100, + .Timeout = 1'000'000'000, + }, + }; + + std::mt19937 mt(seed); + + TSchema fullSchema = TSchema::MakeWithEnoughColumns(maxNumTables); + TString stats = TSchemaStats::MakeRandom(mt, fullSchema, 7, 10).ToJSON(); + + auto kikimr = GetCBOTestsYDB(stats, TDuration::Seconds(10)); + auto db = kikimr.GetQueryClient(); + auto session = db.GetSession().GetValueSync().GetSession(); + + for (int i = 2 /* one table is not possible with all topologies, so 2 */; i < maxNumTables; i ++) { + for (double probability = 0; probability <= 1.0; probability += probabilityStep) { + TRelationGraph graph = Lambda(mt, i, probability); + BenchmarkShuffleEliminationOnTopology(config, session, graph); + } + } + } + + + Y_UNIT_TEST(TRunningStatistics) { + TRunningStatistics stats; + + stats.AddValue(1); + UNIT_ASSERT_EQUAL(stats.GetMedian(), 1); + UNIT_ASSERT_EQUAL(stats.GetMean(), 1 / 1.0); + UNIT_ASSERT_EQUAL(stats.GetN(), 1); + + stats.AddValue(1); + UNIT_ASSERT_EQUAL(stats.GetMedian(), (1 + 1) / 2.0); + UNIT_ASSERT_EQUAL(stats.GetMean(), 2 / 2.0); + UNIT_ASSERT_EQUAL(stats.GetN(), 2); + + stats.AddValue(1); + UNIT_ASSERT_EQUAL(stats.GetMedian(), 1); + UNIT_ASSERT_EQUAL(stats.GetMean(), 3 / 3.0); + UNIT_ASSERT_EQUAL(stats.GetN(), 3); + + stats.AddValue(1); + UNIT_ASSERT_EQUAL(stats.GetMedian(), (1 + 1) / 2.0); + UNIT_ASSERT_EQUAL(stats.GetMean(), 4 / 4.0); + UNIT_ASSERT_EQUAL(stats.GetN(), 4); + + stats.AddValue(3); + UNIT_ASSERT_EQUAL(stats.GetMedian(), 1); + UNIT_ASSERT_EQUAL(stats.GetMean(), 7 / 5.0); + UNIT_ASSERT_EQUAL(stats.GetN(), 5); + + stats.AddValue(5); + UNIT_ASSERT_EQUAL(stats.GetMedian(), (1 + 1) / 2.0); + UNIT_ASSERT_EQUAL(stats.GetMean(), 12 / 6.0); + UNIT_ASSERT_EQUAL(stats.GetN(), 6); + + stats.AddValue(7); + UNIT_ASSERT_EQUAL(stats.GetMedian(), 1); + UNIT_ASSERT_EQUAL(stats.GetMean(), 19 / 7.0); + UNIT_ASSERT_EQUAL(stats.GetN(), 7); + + stats.AddValue(7); + UNIT_ASSERT_EQUAL(stats.GetMedian(), (1 + 3) / 2.0); + UNIT_ASSERT_EQUAL(stats.GetMean(), 26 / 8.0); + UNIT_ASSERT_EQUAL(stats.GetN(), 8); + + stats.AddValue(8); + UNIT_ASSERT_EQUAL(stats.GetMedian(), 3); + UNIT_ASSERT_EQUAL(stats.GetMean(), 34 / 9.0); + UNIT_ASSERT_EQUAL(stats.GetN(), 9); + + stats.AddValue(100); + UNIT_ASSERT_EQUAL(stats.GetMedian(), (3 + 5) / 2.0); + UNIT_ASSERT_EQUAL(stats.GetMean(), 134 / 10.0); + UNIT_ASSERT_EQUAL(stats.GetN(), 10); + } + + Y_UNIT_TEST(ShuffleEliminationBenchmarkOnStar) { + BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/15, /*probabilityStep=*/0.2); + } + + Y_UNIT_TEST(ShuffleEliminationBenchmarkOnLine) { + BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/15, /*probabilityStep=*/0.2); + } + + Y_UNIT_TEST(ShuffleEliminationBenchmarkOnFullyConnected) { + BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/15, /*probabilityStep=*/0.2); + } + + Y_UNIT_TEST(ShuffleEliminationBenchmarkOnRandomTree) { + BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/25, /*probabilityStep=*/0.2); + } + +} + +} +} diff --git a/ydb/core/kqp/ut/join/ya.make b/ydb/core/kqp/ut/join/ya.make index 46c72a6e9621..aaa937d6d5a8 100644 --- a/ydb/core/kqp/ut/join/ya.make +++ b/ydb/core/kqp/ut/join/ya.make @@ -25,6 +25,7 @@ SRCS( kqp_join_ut.cpp kqp_join_order_ut.cpp kqp_join_topology_generator.cpp + kqp_join_topology_ut.cpp ) PEERDIR( From 18ff88f9a924863618f4780972e6a497973f980f Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Tue, 28 Oct 2025 19:08:32 +0000 Subject: [PATCH 14/43] Early exit if values are stable, + stats for derived values, + serialization to CSV --- ydb/core/kqp/ut/common/kqp_benches.h | 160 +++++++++++++++--- ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp | 86 +++++++--- 2 files changed, 199 insertions(+), 47 deletions(-) diff --git a/ydb/core/kqp/ut/common/kqp_benches.h b/ydb/core/kqp/ut/common/kqp_benches.h index 8da40fe8d068..e0cf511ed205 100644 --- a/ydb/core/kqp/ut/common/kqp_benches.h +++ b/ydb/core/kqp/ut/common/kqp_benches.h @@ -108,7 +108,7 @@ class TimeFormatter { template struct TStatistics { - TValue N; + ui64 N; TValue Min, Max; @@ -118,13 +118,22 @@ struct TStatistics { double Mean; double Stdev; - void Dump(IOutputStream &OS) const { - OS << "Median = " << TimeFormatter::Format(Median, MAD) << " (MAD)\n"; - OS << "Mean = " << TimeFormatter::Format(Mean, Stdev) << " (Std. dev.)\n"; - OS << "N = " << N << " [ " << TimeFormatter::Format(Min) << ", " << TimeFormatter::Format(Max) << " ]\n"; - } + double Q1; + double Q3; + + double IQR; + std::vector Outliers; }; +void DumpTimeStatistics(TStatistics stats, IOutputStream &OS) { + OS << "Median = " << TimeFormatter::Format(stats.Median, stats.MAD) << " (MAD)\n"; + + OS << "N = " << stats.N << "\n"; + OS << "Min, Max = [ " << TimeFormatter::Format(stats.Min) << ", " << TimeFormatter::Format(stats.Max) << " ]\n"; + + OS << "Mean = " << TimeFormatter::Format(stats.Mean, stats.Stdev) << " (Std. dev.)\n"; +} + // TValue is arithmetic and can be compared with itself, added to itself, and dividable by floats // TValue is assumed to be cheap to pass by value template @@ -231,13 +240,9 @@ class TRunningStatistics { std::vector deviation; double median = GetMedian(); - for (TValue value : MaxHeap_) { + IterateElements([&](TValue value) { deviation.push_back(std::abs(value - median)); - } - - for (TValue value : MinHeap_) { - deviation.push_back(std::abs(value - median)); - } + }); std::sort(deviation.begin(), deviation.end()); return (deviation[deviation.size() / 2] + deviation[(deviation.size() - 1) / 2]) / 2.0; @@ -263,17 +268,72 @@ class TRunningStatistics { } TStatistics GetStatistics() const { + auto elements = CollectElements(); + std::sort(elements.begin(), elements.end()); + + double q1 = CalculateQuartile(elements, 0.25); + double q3 = CalculateQuartile(elements, 0.75); + double iqr = q3 - q1; + + double median = GetMedian(); + + const double multiplier = 1.5; + double lowerFence = q1 - multiplier * iqr; + double upperFence = q3 + multiplier * iqr; + + std::vector outliers; + for (TValue value : elements) { + if (value < lowerFence || value > upperFence) { + outliers.push_back(value); + } + } + return { .N = GetN(), .Min = GetMin(), .Max = GetMax(), - .Median = GetMedian(), + .Median = median, .MAD = CalculateMAD(), .Mean = GetMean(), .Stdev = CalculateStdDev(), + .Q1 = q1, + .Q3 = q3, + .IQR = iqr, + .Outliers = outliers }; } + std::vector CollectElements() const { + std::vector values; + IterateElements([&](TValue value) { + values.push_back(value); + }); + + return values; + } + + TRunningStatistics operator-(TRunningStatistics rhsStats) const { + TRunningStatistics stats; + IterateElements([&](TValue lhs) { + rhsStats.IterateElements([&](TValue rhs) { + stats.AddValue((i64) lhs - (i64) rhs); + }); + }); + + return stats; + } + + TRunningStatistics operator/(TRunningStatistics rhsStats) const { + TRunningStatistics stats; + IterateElements([&](TValue lhs) { + rhsStats.IterateElements([&](TValue rhs) { + stats.AddValue((double) lhs / (double) rhs); + }); + }); + + return stats; + } + private: std::vector MaxHeap_; std::vector MinHeap_; @@ -286,8 +346,63 @@ class TRunningStatistics { std::optional Min_; std::optional Max_; + +private: + template + void IterateElements(Lambda &&lambda) const { + for (TValue value : MaxHeap_) { + lambda(value); + } + + for (TValue value : MinHeap_) { + lambda(value); + } + } + + static double CalculateQuartile(const std::vector& data, double percentile) { // TODO: rename + assert(percentile >= 0.0 && percentile <= 1.0); + assert(std::is_sorted(data.begin(), data.end())); + + double position = percentile * (data.size() - 1); + int lowerIndex = floor(position); + int upperIndex = ceil(position); + + if (lowerIndex == upperIndex) { + return data[lowerIndex]; + } + + double weight = position - lowerIndex; + return data[lowerIndex] * (1 - weight) + data[upperIndex] * weight; + } }; +template +static std::string joinVector(const std::vector &data) { + if (data.empty()) { + return ""; + } + + std::string str; + str += std::to_string(data[0]); + + for (ui32 i = 1; i < data.size(); ++ i) { + str += ";" + std::to_string(data[i]); + } + + return str; +} + + +void DumpBoxPlotCSVHeader(IOutputStream &OS) { + OS << "N,median,Q1,Q3,IQR,MAD,outliers\n"; +} + +template +void DumpBoxPlotToCSV(IOutputStream &OS, ui32 i, TStatistics stat) { + OS << i << "," << stat.Median << "," << stat.Q1 << "," << stat.Q3 << "," << stat.IQR << "," << stat.MAD << "," << joinVector(stat.Outliers) << "\n"; +} + + struct TRepeatedTestConfig { ui32 MinRepeats; ui32 MaxRepeats; @@ -295,7 +410,7 @@ struct TRepeatedTestConfig { }; template -std::optional> RepeatedTest(TRepeatedTestConfig config, TLambda &&lambda) { +std::optional> RepeatedTest(TRepeatedTestConfig config, ui64 singleRunTimeout, double thresholdMAD, TLambda &&lambda) { assert(config.MinRepeats >= 1); TRunningStatistics stats; @@ -303,34 +418,37 @@ std::optional> RepeatedTest(TRepeatedTestConfig config, TLambd bool hasTimedOut = false; ui64 ellapsedTime = MeasureTimeNanos([&]() { hasTimedOut = !lambda(); }); - if (hasTimedOut) { + if (hasTimedOut || ellapsedTime > singleRunTimeout) { return std::nullopt; } stats.AddValue(ellapsedTime); - } while (stats.GetN() < config.MaxRepeats && + } while ((stats.GetN() < config.MaxRepeats) && (stats.GetN() < config.MinRepeats || - stats.GetTotal() + stats.GetMedian() <= config.Timeout)); + (stats.GetTotal() + stats.GetMedian() <= config.Timeout && + stats.CalculateMAD() / stats.GetMedian() > thresholdMAD))); - return stats.GetStatistics(); + return stats; } struct TBenchmarkConfig { TRepeatedTestConfig Warmup; TRepeatedTestConfig Bench; + ui64 SingleRunTimeout; + double MADThreshold; }; template -std::optional> Benchmark(TBenchmarkConfig config, Lambda &&lambda) { +std::optional> Benchmark(TBenchmarkConfig config, Lambda &&lambda) { if (config.Warmup.MaxRepeats != 0) { - auto warmupStats = RepeatedTest(config.Warmup, lambda); + auto warmupStats = RepeatedTest(config.Warmup, config.SingleRunTimeout, config.MADThreshold, lambda); if (!warmupStats) { return std::nullopt; } } - return RepeatedTest(config.Bench, lambda); + return RepeatedTest(config.Bench, config.SingleRunTimeout, config.MADThreshold, lambda); } } // end namespace NKikimr::NKqp diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp index 41a958d59dd2..2680fb7368ca 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp @@ -54,18 +54,18 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { } std::string ConfigureQuery(const std::string &query, bool enableShuffleElimination = false, unsigned optLevel = 2) { - std::string queryWithShuffleElimination = "PRAGMA ydb.OptShuffleElimination = \""; + std::string queryWithShuffleElimination = "PRAGMA ydb.OptShuffleElimination=\""; queryWithShuffleElimination += enableShuffleElimination ? "true" : "false"; queryWithShuffleElimination += "\";\n"; queryWithShuffleElimination += "PRAGMA ydb.MaxDPHypDPTableSize='4294967295';\n"; queryWithShuffleElimination += "PRAGMA ydb.ForceShuffleElimination='true';\n"; - queryWithShuffleElimination += "PRAGMA ydb.CostBasedOptimizationLevel = \"" + std::to_string(optLevel) + "\";\n"; + queryWithShuffleElimination += "PRAGMA ydb.CostBasedOptimizationLevel=\"" + std::to_string(optLevel) + "\";\n"; queryWithShuffleElimination += query; return queryWithShuffleElimination; } - std::optional> BenchmarkExplain(TBenchmarkConfig config, NYdb::NQuery::TSession session, const TString& query) { + std::optional> BenchmarkExplain(TBenchmarkConfig config, NYdb::NQuery::TSession session, const TString& query) { std::optional savedPlan = std::nullopt; auto stats = Benchmark(config, [&]() -> bool { auto plan = ExplainQuery(session, query); @@ -85,22 +85,47 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { JustPrintPlan(*savedPlan); Cout << "--------------------------------------------------------------------------\n"; - stats->Dump(Cout); + DumpTimeStatistics(stats->GetStatistics(), Cout); return stats; } - std::optional> BenchmarkShuffleElimination(TBenchmarkConfig config, NYdb::NQuery::TSession session, const TString& query) { + struct ShuffleEliminationBenchmarkResult { + TRunningStatistics WithoutCBO; + TRunningStatistics WithShuffleElimination; + TRunningStatistics WithoutShuffleElimination; + }; + + std::optional> BenchmarkShuffleElimination(TBenchmarkConfig config, NYdb::NQuery::TSession session, const TString& query) { Cout << "--------------------------------- W/O CBO --------------------------------\n"; - BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/false, /*optLevel=*/1)); + auto withoutCBO = BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/false, /*optLevel=*/0)); + if (!withoutCBO) { + return std::nullopt; + } + Cout << "--------------------------------- CBO-SE ---------------------------------\n"; - BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/false, /*optLevel=*/2)); + auto withoutShuffleElimination = BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/false, /*optLevel=*/2)); + if (!withoutShuffleElimination) { + return std::nullopt; + } + Cout << "--------------------------------- CBO+SE ---------------------------------\n"; - auto result = BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/true, /*optLevel=*/2)); + auto withShuffleElimination = BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/true, /*optLevel=*/2)); + if (!withShuffleElimination) { + return std::nullopt; + } + Cout << "--------------------------------------------------------------------------\n"; - return result; + + ShuffleEliminationBenchmarkResult result { + .WithoutCBO = *withoutCBO, + .WithShuffleElimination = *withShuffleElimination - *withoutCBO, + .WithoutShuffleElimination = *withoutShuffleElimination - *withoutCBO + }; + + return result.WithShuffleElimination / result.WithoutShuffleElimination; } TKikimrRunner GetCBOTestsYDB(TString stats, TDuration compilationTimeout) { @@ -125,7 +150,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { return TKikimrRunner(serverSettings); } - bool BenchmarkShuffleEliminationOnTopology(TBenchmarkConfig config, NYdb::NQuery::TSession session, TRelationGraph graph) { + std::optional> BenchmarkShuffleEliminationOnTopology(TBenchmarkConfig config, NYdb::NQuery::TSession session, TRelationGraph graph) { Cout << "\n\n"; Cout << "================================= CREATE =================================\n"; graph.DumpGraph(Cout); @@ -138,7 +163,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { auto creationQuery = graph.GetSchema().MakeCreateQuery(); Cout << creationQuery; if (!ExecuteQuery(session, creationQuery)) { - return false; + return std::nullopt; } Cout << "================================= BENCHMARK ==============================\n"; @@ -150,15 +175,11 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { auto deletionQuery = graph.GetSchema().MakeDropQuery(); Cout << deletionQuery; if (!ExecuteQuery(session, deletionQuery)) { // TODO: this is really bad probably? - return false; + return std::nullopt; } Cout << "==========================================================================\n"; - if (!resultTime) { - return false; // query timed out - } - - return true; + return resultTime; } template @@ -166,15 +187,18 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { auto config = TBenchmarkConfig { .Warmup = { .MinRepeats = 1, - .MaxRepeats = 3, - .Timeout = 10'000'000, + .MaxRepeats = 5, + .Timeout = 1'000'000'000, }, .Bench = { - .MinRepeats = 3, - .MaxRepeats = 100, - .Timeout = 1'000'000'000, + .MinRepeats = 10, + .MaxRepeats = 30, + .Timeout = 10'000'000'000, }, + + .SingleRunTimeout = 20'000'000'000, + .MADThreshold = 0.05 }; std::mt19937 mt(seed); @@ -186,12 +210,22 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { auto db = kikimr.GetQueryClient(); auto session = db.GetSession().GetValueSync().GetSession(); - for (int i = 2 /* one table is not possible with all topologies, so 2 */; i < maxNumTables; i ++) { - for (double probability = 0; probability <= 1.0; probability += probabilityStep) { - TRelationGraph graph = Lambda(mt, i, probability); - BenchmarkShuffleEliminationOnTopology(config, session, graph); + auto OS = TUnbufferedFileOutput("results.csv"); + DumpBoxPlotCSVHeader(OS); + + for (int i = 2 /* one table is not possible with all topologies, so 2 */; true; i ++) { + (void) probabilityStep; + double probability = 0.5; + + TRelationGraph graph = Lambda(mt, i, probability); + auto result = BenchmarkShuffleEliminationOnTopology(config, session, graph); + if (!result) { + goto stop; } + + DumpBoxPlotToCSV(OS, i, result->GetStatistics()); } + stop:; } From ab8db74b09613a68f0540254ad7a534f943944bd Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Wed, 29 Oct 2025 16:10:15 +0000 Subject: [PATCH 15/43] Implement runtime configuration of benchmarks --- ydb/core/kqp/ut/common/kqp_arg_parser.h | 215 ++++++++++++++++++ ydb/core/kqp/ut/common/kqp_benches.h | 8 +- ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp | 103 +++++++-- 3 files changed, 298 insertions(+), 28 deletions(-) create mode 100644 ydb/core/kqp/ut/common/kqp_arg_parser.h diff --git a/ydb/core/kqp/ut/common/kqp_arg_parser.h b/ydb/core/kqp/ut/common/kqp_arg_parser.h new file mode 100644 index 000000000000..b94aad8d1ccd --- /dev/null +++ b/ydb/core/kqp/ut/common/kqp_arg_parser.h @@ -0,0 +1,215 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace NKikimr::NKqp { + + +class TArgs { +public: + template + class TRangedValueIter { + public: + TRangedValueIter(TValue current, TValue end, TValue step) + : Current_(current) + , End_(end) + , Step_(step) + { + } + + TValue operator*() const { + return Current_; + } + + TRangedValueIter& operator++() { + Current_ += Step_; + if (Current_ >= End_) { + Current_ = End_; + } + + return *this; + } + + bool operator!=(TRangedValueIter other) const { + assert(Step_ == other.Step_); + return Current_ != other.Current_; + } + + private: + TValue Current_; + TValue End_; + TValue Step_; + }; + + template + class TRangedValue { + public: + TRangedValue(TValue from, TValue to, TValue step) + : IsRange_(true) + , From_(from) + , To_(to) + , Step_(step) + { + } + + TRangedValue(TValue from) + : IsRange_(false) + , From_(from) + , To_(from) + , Step_(1) + { + } + + bool IsRange() const { + return IsRange_; + } + + TRangedValueIter end() const { + TValue End = To_ + 1; // immediately after the last + return TRangedValueIter{End, End, Step_}; + } + + TRangedValueIter begin() const { + return TRangedValueIter{From_, *end(), Step_}; + } + + TValue GetValue() const { + return From_; + } + + TValue GetFirst() const { + return From_; + } + + TValue GetLast() const { + return To_; + } + + TValue GetStep() const { + return Step_; + } + + private: + bool IsRange_; + + TValue From_; + TValue To_; + TValue Step_; + }; + +public: + TArgs(std::string input) + : Values_(ParseMap(input)) + { + } + + template + auto GetArg(std::string key) { + Cout << key << " " << Values_[key] << "\n"; + return ParseRangedValue(Values_[key]); + } + + bool HasArg(std::string key) { + return Values_.contains(key); + } + + +private: + std::map Values_; + +private: + static std::map ParseMap(const std::string& input, char delimiter = ';') { + std::map result; + std::stringstream ss(input); + + std::string entry; + while (std::getline(ss, entry, delimiter)) { + // each entry looks like key value pair, e.g. "N=5" + size_t pos = entry.find('='); + + if (pos != std::string::npos) { + result[entry.substr(0, pos)] = entry.substr(pos + 1); + } + } + + return result; + } + + template + static auto ParseRangedValue(const std::string& input) { + // Check if it contains ".." + size_t dotdot = input.find(".."); + + if (dotdot == std::string::npos) { + // parse fixed value + auto value = ParseValue(input); + return TRangedValue{value}; + } else { + // parse ranged (with step or without) + size_t comma = input.find(','); + + auto to = ParseValue(input.substr(dotdot + 2)); + if (comma != std::string::npos && comma < dotdot) { + // parse ranges like "0.1,0.2..1.0" + auto first = ParseValue(input.substr(0, comma)); + auto second = ParseValue(input.substr(comma + 1, dotdot - comma - 1)); + auto step = second - first; + return TRangedValue{first, to, step}; + } + + // parse ranges like "1..100" + auto first = ParseValue(input.substr(0, dotdot)); + return TRangedValue{first, to, /*default step=*/1}; + } + } + + template + static auto ParseValue(const std::string& input) { + if constexpr (std::is_same_v) { + return std::stod(input); + } else if constexpr (std::is_same_v) { + return static_cast(std::stoull(input)); + } else if constexpr (std::is_same_v) { + return static_cast(std::stoll(input)); + } else if constexpr (std::is_same_v) { + return input; + } else if constexpr (std::is_same_v) { + return static_cast(ParseDuration(input).count()); + } else { + static_assert(false, "Unhandled type"); + } + } + + static std::chrono::nanoseconds ParseDuration(const std::string& input) { + std::regex pattern(R"((\d+(?:\.\d+)?)\s*(ns|us|ms|s|m|h))"); + std::smatch match; + + if (!std::regex_match(input, match, pattern)) { + throw std::invalid_argument("Invalid duration format"); + } + + double value = std::stod(match[1]); + std::string unit = match[2]; + + if (unit == "ns") return std::chrono::nanoseconds(static_cast(value)); + if (unit == "us") return std::chrono::microseconds(static_cast(value)); + if (unit == "ms") return std::chrono::milliseconds(static_cast(value)); + if (unit == "s") return std::chrono::seconds(static_cast(value)); + if (unit == "m") return std::chrono::minutes(static_cast(value)); + if (unit == "h") return std::chrono::hours(static_cast(value)); + + throw std::invalid_argument("Unknown unit"); + } + +}; + + +} diff --git a/ydb/core/kqp/ut/common/kqp_benches.h b/ydb/core/kqp/ut/common/kqp_benches.h index e0cf511ed205..c2b314ef8bb7 100644 --- a/ydb/core/kqp/ut/common/kqp_benches.h +++ b/ydb/core/kqp/ut/common/kqp_benches.h @@ -26,7 +26,7 @@ ui64 MeasureTimeNanos(Lambda &&lambda) { class TimeFormatter { public: - static std::string Format(uint64_t valueNs, uint64_t uncertaintyNs) { + static std::string Format(uint64_t valueNs, ui64 uncertaintyNs) { auto [unit, scale] = SelectUnit(valueNs); double scaledValue = valueNs / scale; @@ -55,7 +55,7 @@ class TimeFormatter { } private: - static std::pair SelectUnit(uint64_t nanoseconds) { + static std::pair SelectUnit(ui64 nanoseconds) { if (nanoseconds >= 1'000'000'000) { return {"s", 1e9}; } else if (nanoseconds >= 1'000'000) { @@ -404,8 +404,8 @@ void DumpBoxPlotToCSV(IOutputStream &OS, ui32 i, TStatistics stat) { struct TRepeatedTestConfig { - ui32 MinRepeats; - ui32 MaxRepeats; + ui64 MinRepeats; + ui64 MaxRepeats; ui64 Timeout; }; diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp index 2680fb7368ca..a8467ce73f11 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp @@ -1,10 +1,14 @@ +#include #include #include +#include #include #include #include +#include + #include "kqp_join_topology_generator.h" namespace NKikimr { @@ -182,9 +186,22 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { return resultTime; } - template - void BenchmarkShuffleEliminationOnTopologies(int maxNumTables, double probabilityStep, unsigned seed = 0) { - auto config = TBenchmarkConfig { + + template + void OverrideWithArg(std::string key, TArgs args, auto& value) { + if (args.HasArg(key)) { + value = args.GetArg(key).GetValue(); + } + } + + void OverrideRepeatedTestConfig(std::string prefix, TArgs args, TRepeatedTestConfig &config) { + OverrideWithArg(prefix + ".MinRepeats", args, config.MinRepeats); + OverrideWithArg(prefix + ".MaxRepeats", args, config.MaxRepeats); + OverrideWithArg(prefix + ".Timeout", args, config.Timeout); + } + + TBenchmarkConfig GetBenchmarkConfig(TArgs args, std::string prefix = "config") { + TBenchmarkConfig config = /*default=*/{ .Warmup = { .MinRepeats = 1, .MaxRepeats = 5, @@ -201,9 +218,36 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { .MADThreshold = 0.05 }; + OverrideRepeatedTestConfig(prefix + ".Warmup", args, config.Warmup); + OverrideRepeatedTestConfig(prefix + ".Bench", args, config.Bench); + OverrideWithArg(prefix + ".MADThreshold", args, config.MADThreshold); + OverrideWithArg(prefix + ".SingleRunTimeout", args, config.SingleRunTimeout); + + return config; + } + + void DumpBenchmarkConfig(IOutputStream &OS, TBenchmarkConfig config) { + OS << "config = {\n"; + OS << " .Warmup = {\n"; + OS << " .MinRepeats = " << config.Warmup.MinRepeats << ",\n"; + OS << " .MaxRepeats = " << config.Warmup.MaxRepeats << ",\n"; + OS << " .Timeout = " << TimeFormatter::Format(config.Warmup.Timeout) << "\n"; + OS << " },\n\n"; + OS << " .Bench = {\n"; + OS << " .MinRepeats = " << config.Bench.MinRepeats << ",\n"; + OS << " .MaxRepeats = " << config.Bench.MaxRepeats << ",\n"; + OS << " .Timeout = " << TimeFormatter::Format(config.Bench.Timeout) << "\n"; + OS << " },\n\n"; + OS << " .SingleRunTimeout = " << TimeFormatter::Format(config.SingleRunTimeout) << ",\n"; + OS << " .MADThreshold = " << config.MADThreshold << "\n"; + OS << "}\n"; + } + + template + void BenchmarkShuffleEliminationOnTopologies(TBenchmarkConfig config, TArgs::TRangedValue numTables, TArgs::TRangedValue sameColumnProbability, unsigned seed = 0) { std::mt19937 mt(seed); - TSchema fullSchema = TSchema::MakeWithEnoughColumns(maxNumTables); + TSchema fullSchema = TSchema::MakeWithEnoughColumns(numTables.GetLast()); TString stats = TSchemaStats::MakeRandom(mt, fullSchema, 7, 10).ToJSON(); auto kikimr = GetCBOTestsYDB(stats, TDuration::Seconds(10)); @@ -213,17 +257,16 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { auto OS = TUnbufferedFileOutput("results.csv"); DumpBoxPlotCSVHeader(OS); - for (int i = 2 /* one table is not possible with all topologies, so 2 */; true; i ++) { - (void) probabilityStep; - double probability = 0.5; + for (ui64 n : numTables) { + for (double probability : sameColumnProbability) { + TRelationGraph graph = Lambda(mt, n, probability); + auto result = BenchmarkShuffleEliminationOnTopology(config, session, graph); + if (!result) { + goto stop; + } - TRelationGraph graph = Lambda(mt, i, probability); - auto result = BenchmarkShuffleEliminationOnTopology(config, session, graph); - if (!result) { - goto stop; + DumpBoxPlotToCSV(OS, n, result->GetStatistics()); } - - DumpBoxPlotToCSV(OS, i, result->GetStatistics()); } stop:; } @@ -283,20 +326,32 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { UNIT_ASSERT_EQUAL(stats.GetN(), 10); } - Y_UNIT_TEST(ShuffleEliminationBenchmarkOnStar) { - BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/15, /*probabilityStep=*/0.2); - } + // Y_UNIT_TEST(ShuffleEliminationBenchmarkOnStar) { + // BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/15, /*probabilityStep=*/0.2); + // } - Y_UNIT_TEST(ShuffleEliminationBenchmarkOnLine) { - BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/15, /*probabilityStep=*/0.2); - } + // Y_UNIT_TEST(ShuffleEliminationBenchmarkOnLine) { + // BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/15, /*probabilityStep=*/0.2); + // } - Y_UNIT_TEST(ShuffleEliminationBenchmarkOnFullyConnected) { - BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/15, /*probabilityStep=*/0.2); - } + // Y_UNIT_TEST(ShuffleEliminationBenchmarkOnFullyConnected) { + // BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/15, /*probabilityStep=*/0.2); + // } + + // Y_UNIT_TEST(ShuffleEliminationBenchmarkOnRandomTree) { + // BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/25, /*probabilityStep=*/0.2); + // } + + Y_UNIT_TEST(Benchmark) { + TArgs args{GetTestParam("TOPOLOGY")}; + + auto config = GetBenchmarkConfig(args); + DumpBenchmarkConfig(Cout, config); - Y_UNIT_TEST(ShuffleEliminationBenchmarkOnRandomTree) { - BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/25, /*probabilityStep=*/0.2); + BenchmarkShuffleEliminationOnTopologies( + config, + /*maxNumTables=*/args.GetArg("N"), + /*probabilityStep=*/args.GetArg("P")); } } From e747ec2167309363e8a809869330e906a2d730d7 Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Wed, 29 Oct 2025 16:37:16 +0000 Subject: [PATCH 16/43] Simplify test reproduction by wrapping mersenne twister --- ydb/core/kqp/ut/common/kqp_serializable_rng.h | 102 ++++++++++++++++++ .../ut/join/kqp_join_topology_generator.cpp | 24 ++--- .../kqp/ut/join/kqp_join_topology_generator.h | 22 ++-- ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp | 6 +- 4 files changed, 131 insertions(+), 23 deletions(-) create mode 100644 ydb/core/kqp/ut/common/kqp_serializable_rng.h diff --git a/ydb/core/kqp/ut/common/kqp_serializable_rng.h b/ydb/core/kqp/ut/common/kqp_serializable_rng.h new file mode 100644 index 000000000000..9065007af491 --- /dev/null +++ b/ydb/core/kqp/ut/common/kqp_serializable_rng.h @@ -0,0 +1,102 @@ +#pragma once + +#include +#include +#include + + +namespace NKikimr::NKqp { + + +// wrapper around std::mt19937 that tracks usage and simplifies serialization +class TSerializableMT19937 { +public: // compatibility with std::mt19937 + using result_type = std::mt19937::result_type; + static constexpr auto default_seed = std::mt19937::default_seed; + +public: + TSerializableMT19937() + : TSerializableMT19937(default_seed) + { + } + + TSerializableMT19937(uint32_t seed) + : Engine_(seed) + , Seed_(seed) + , Counter_(0) + { + } + + uint64_t Serialize() const { + return (static_cast(Seed_) << 32ULL) | static_cast(Counter_); + } + + void Restore(uint64_t state) { + Seed_ = static_cast(state >> 32); + Counter_ = static_cast(state & 0xFFFFFFFF); + + Engine_.seed(Seed_); + Engine_.discard(Counter_); + } + + uint32_t GetCounter() const { + return Counter_; + } + + uint32_t GetSeed() const { + return Seed_; + } + + static TSerializableMT19937 Deserialize(uint64_t key) { + TSerializableMT19937 mt; + mt.Restore(key); + + return mt; + } + +public: // compatibility with std::mt19937 + static constexpr auto min() { return std::mt19937::min(); } + static constexpr auto max() { return std::mt19937::max(); } + + auto operator()() { + assert(Counter_ != UINT32_MAX); + ++ Counter_; + return Engine_(); + } + + void seed(uint32_t seed) { + Seed_ = seed; + Counter_ = 0; + + Engine_.seed(seed); + } + + void discard(uint64_t n) { + assert(n <= static_cast(UINT32_MAX - Counter_)); + + Counter_ += static_cast(n); + Engine_.discard(n); + } + + void reset() { + Counter_ = 0; + Engine_.seed(Seed_); + } + + bool operator==(const TSerializableMT19937& other) const { + return Seed_ == other.Seed_ && Counter_ == other.Counter_; + } + + bool operator!=(const TSerializableMT19937& other) const { + return !(*this == other); + } + +private: + std::mt19937 Engine_; + uint32_t Seed_; + uint32_t Counter_; + +}; + +} + diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp index 912098063384..ed2e88460881 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp @@ -66,18 +66,18 @@ static std::string getTablePath(unsigned tableID) { } -static unsigned getRandom(std::mt19937 &mt, unsigned a, unsigned b) { +static unsigned getRandom(TRNG &mt, unsigned a, unsigned b) { std::uniform_int_distribution<> distribution(a, b); return distribution(mt); } -static unsigned getRandomBool(std::mt19937 &mt, double trueProbability) { +static unsigned getRandomBool(TRNG &mt, double trueProbability) { std::uniform_real_distribution<> distribution(0.0, 1.0); return distribution(mt) < trueProbability; } -unsigned TTable::GetRandomOrNewColumn(std::mt19937 &mt, double newColumnProbability) { +unsigned TTable::GetRandomOrNewColumn(TRNG &mt, double newColumnProbability) { bool generateNewColumn = getRandomBool(mt, newColumnProbability); if (generateNewColumn || NumColumns == 0) { return NumColumns ++; @@ -86,7 +86,7 @@ unsigned TTable::GetRandomOrNewColumn(std::mt19937 &mt, double newColumnProbabil return GetRandomColumn(mt); } -unsigned TTable::GetRandomColumn(std::mt19937 &mt) const { +unsigned TTable::GetRandomColumn(TRNG &mt) const { assert(NumColumns != 0); return getRandom(mt, 0, NumColumns - 1); } @@ -132,7 +132,7 @@ void TSchema::Rename(std::vector oldToNew) { } -TRelationGraph TRelationGraph::FromPrufer(std::mt19937 &mt, const std::vector& prufer, double newColumnProbability) { +TRelationGraph TRelationGraph::FromPrufer(TRNG &mt, const std::vector& prufer, double newColumnProbability) { unsigned n = prufer.size() + 2; std::vector degree(n, 1); @@ -170,7 +170,7 @@ TRelationGraph TRelationGraph::FromPrufer(std::mt19937 &mt, const std::vector &oldToNew) { } -TSchemaStats TSchemaStats::MakeRandom(std::mt19937 &mt, const TSchema &schema, unsigned a, unsigned b) { +TSchemaStats TSchemaStats::MakeRandom(TRNG &mt, const TSchema &schema, unsigned a, unsigned b) { std::uniform_int_distribution<> distribution(a, b); std::vector stats(schema.GetSize()); @@ -332,7 +332,7 @@ std::string TSchemaStats::ToJSON() const { return ss.str(); } -TRelationGraph GenerateLine(std::mt19937 &mt, unsigned numNodes, double newColumnProbability) { +TRelationGraph GenerateLine(TRNG &mt, unsigned numNodes, double newColumnProbability) { TRelationGraph graph(numNodes); unsigned lastVertex = 0; @@ -349,7 +349,7 @@ TRelationGraph GenerateLine(std::mt19937 &mt, unsigned numNodes, double newColum return graph; } -TRelationGraph GenerateStar(std::mt19937 &mt, unsigned numNodes, double newColumnProbability) { +TRelationGraph GenerateStar(TRNG &mt, unsigned numNodes, double newColumnProbability) { TRelationGraph graph(numNodes); unsigned root = 0; @@ -360,7 +360,7 @@ TRelationGraph GenerateStar(std::mt19937 &mt, unsigned numNodes, double newColum return graph; } -TRelationGraph GenerateFullyConnected(std::mt19937 &mt, unsigned numNodes, +TRelationGraph GenerateFullyConnected(TRNG &mt, unsigned numNodes, double newColumnProbability) { TRelationGraph graph(numNodes); @@ -374,7 +374,7 @@ TRelationGraph GenerateFullyConnected(std::mt19937 &mt, unsigned numNodes, return graph; } -static std::vector GenerateRandomPruferSequence(std::mt19937 &mt, unsigned numNodes) { +static std::vector GenerateRandomPruferSequence(TRNG &mt, unsigned numNodes) { assert(numNodes >= 2); std::uniform_int_distribution<> distribution(0, numNodes - 1); @@ -386,7 +386,7 @@ static std::vector GenerateRandomPruferSequence(std::mt19937 &mt, unsi return prufer; } -TRelationGraph GenerateRandomTree(std::mt19937 &mt, unsigned numNodes, double newColumnProbability) { +TRelationGraph GenerateRandomTree(TRNG &mt, unsigned numNodes, double newColumnProbability) { auto prufer = GenerateRandomPruferSequence(mt, numNodes); return TRelationGraph::FromPrufer(mt, prufer, newColumnProbability); } diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_generator.h b/ydb/core/kqp/ut/join/kqp_join_topology_generator.h index 25a5a04cbd8e..f89e9ec0ac55 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_generator.h +++ b/ydb/core/kqp/ut/join/kqp_join_topology_generator.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -10,6 +11,9 @@ namespace NKikimr::NKqp { + +using TRNG = TSerializableMT19937; + class TTable { public: TTable(unsigned numColumns = 0) @@ -17,14 +21,14 @@ class TTable { { } - unsigned GetRandomOrNewColumn(std::mt19937 &mt, double newColumnProbability); + unsigned GetRandomOrNewColumn(TRNG &mt, double newColumnProbability); unsigned GetNumColumns() const { return NumColumns; } private: - unsigned GetRandomColumn(std::mt19937 &mt) const; + unsigned GetRandomColumn(TRNG &mt) const; private: // table has columns from 0..NumColumns @@ -72,9 +76,9 @@ class TRelationGraph { { } - static TRelationGraph FromPrufer(std::mt19937 &mt, const std::vector& prufer, double newColumnProbability); + static TRelationGraph FromPrufer(TRNG &mt, const std::vector& prufer, double newColumnProbability); - void Connect(std::mt19937 &mt, unsigned lhs, unsigned rhs, double newColumnProbability); + void Connect(TRNG &mt, unsigned lhs, unsigned rhs, double newColumnProbability); std::string MakeQuery() const; @@ -121,7 +125,7 @@ class TSchemaStats { { } - static TSchemaStats MakeRandom(std::mt19937 &mt, const TSchema &schema, unsigned a, unsigned b); + static TSchemaStats MakeRandom(TRNG &mt, const TSchema &schema, unsigned a, unsigned b); std::string ToJSON() const; @@ -130,10 +134,10 @@ class TSchemaStats { }; -TRelationGraph GenerateLine(std::mt19937 &mt, unsigned numNodes, double newColumnProbability); -TRelationGraph GenerateStar(std::mt19937 &mt, unsigned numNodes, double newColumnProbability); -TRelationGraph GenerateFullyConnected(std::mt19937 &mt, unsigned numNodes, double newColumnProbability); -TRelationGraph GenerateRandomTree(std::mt19937 &mt, unsigned numNodes, double newColumnProbability); +TRelationGraph GenerateLine(TRNG &mt, unsigned numNodes, double newColumnProbability); +TRelationGraph GenerateStar(TRNG &mt, unsigned numNodes, double newColumnProbability); +TRelationGraph GenerateFullyConnected(TRNG &mt, unsigned numNodes, double newColumnProbability); +TRelationGraph GenerateRandomTree(TRNG &mt, unsigned numNodes, double newColumnProbability); } // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp index a8467ce73f11..95fa02cdcd6d 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -11,6 +10,7 @@ #include "kqp_join_topology_generator.h" + namespace NKikimr { namespace NKqp { @@ -245,7 +245,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { template void BenchmarkShuffleEliminationOnTopologies(TBenchmarkConfig config, TArgs::TRangedValue numTables, TArgs::TRangedValue sameColumnProbability, unsigned seed = 0) { - std::mt19937 mt(seed); + TRNG mt(seed); TSchema fullSchema = TSchema::MakeWithEnoughColumns(numTables.GetLast()); TString stats = TSchemaStats::MakeRandom(mt, fullSchema, 7, 10).ToJSON(); @@ -259,6 +259,8 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { for (ui64 n : numTables) { for (double probability : sameColumnProbability) { + Cout << "Seed: " << mt.Serialize() << "\n"; + TRelationGraph graph = Lambda(mt, n, probability); auto result = BenchmarkShuffleEliminationOnTopology(config, session, graph); if (!result) { From da59890db97be77e119c22a1d5a970eea04736f1 Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Wed, 29 Oct 2025 20:42:52 +0000 Subject: [PATCH 17/43] Allow args to be separated by space --- ydb/core/kqp/ut/common/kqp_arg_parser.h | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/ydb/core/kqp/ut/common/kqp_arg_parser.h b/ydb/core/kqp/ut/common/kqp_arg_parser.h index b94aad8d1ccd..f82f9e8770df 100644 --- a/ydb/core/kqp/ut/common/kqp_arg_parser.h +++ b/ydb/core/kqp/ut/common/kqp_arg_parser.h @@ -126,6 +126,23 @@ class TArgs { std::map Values_; private: + static void LTrim(std::string &input) { + input.erase(input.begin(), std::find_if(input.begin(), input.end(), [](unsigned char ch) { + return !std::isspace(ch); + })); + } + + static void RTrim(std::string &input) { + input.erase(std::find_if(input.rbegin(), input.rend(), [](unsigned char ch) { + return !std::isspace(ch); + }).base(), input.end()); + } + + static void Trim(std::string &input) { + LTrim(input); + RTrim(input); + } + static std::map ParseMap(const std::string& input, char delimiter = ';') { std::map result; std::stringstream ss(input); @@ -133,10 +150,14 @@ class TArgs { std::string entry; while (std::getline(ss, entry, delimiter)) { // each entry looks like key value pair, e.g. "N=5" + Trim(entry); size_t pos = entry.find('='); if (pos != std::string::npos) { - result[entry.substr(0, pos)] = entry.substr(pos + 1); + std::string key = entry.substr(0, pos); + std::string value = entry.substr(pos + 1); + Trim(value); + result[std::move(key)] = std::move(value); } } From 92664a7114678e501634cffb5a43a3169ac580aa Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Wed, 29 Oct 2025 20:43:10 +0000 Subject: [PATCH 18/43] Simplify topology reproduction --- ydb/core/kqp/ut/common/kqp_arg_parser.h | 1 - ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp | 22 ++++++++++++++----- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/ydb/core/kqp/ut/common/kqp_arg_parser.h b/ydb/core/kqp/ut/common/kqp_arg_parser.h index f82f9e8770df..d6047f88df7e 100644 --- a/ydb/core/kqp/ut/common/kqp_arg_parser.h +++ b/ydb/core/kqp/ut/common/kqp_arg_parser.h @@ -113,7 +113,6 @@ class TArgs { template auto GetArg(std::string key) { - Cout << key << " " << Values_[key] << "\n"; return ParseRangedValue(Values_[key]); } diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp index 95fa02cdcd6d..de4062290f22 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp @@ -155,7 +155,6 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { } std::optional> BenchmarkShuffleEliminationOnTopology(TBenchmarkConfig config, NYdb::NQuery::TSession session, TRelationGraph graph) { - Cout << "\n\n"; Cout << "================================= CREATE =================================\n"; graph.DumpGraph(Cout); @@ -244,12 +243,15 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { } template - void BenchmarkShuffleEliminationOnTopologies(TBenchmarkConfig config, TArgs::TRangedValue numTables, TArgs::TRangedValue sameColumnProbability, unsigned seed = 0) { - TRNG mt(seed); + void BenchmarkShuffleEliminationOnTopologies(TBenchmarkConfig config, TArgs::TRangedValue numTables, TArgs::TRangedValue sameColumnProbability, uint64_t state = 0) { + TRNG mt = TRNG::Deserialize(state); + mt.reset(); TSchema fullSchema = TSchema::MakeWithEnoughColumns(numTables.GetLast()); TString stats = TSchemaStats::MakeRandom(mt, fullSchema, 7, 10).ToJSON(); + mt.Restore(state); + auto kikimr = GetCBOTestsYDB(stats, TDuration::Seconds(10)); auto db = kikimr.GetQueryClient(); auto session = db.GetSession().GetValueSync().GetSession(); @@ -259,8 +261,9 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { for (ui64 n : numTables) { for (double probability : sameColumnProbability) { - Cout << "Seed: " << mt.Serialize() << "\n"; - + Cout << "\n\n"; + Cout << "================================ REPRODUCE -------------------------------\n"; + Cout << "KQP_TOPOLOGY='N=" << n << "; P=" << probability << "; seed=" << mt.Serialize() << "'\n"; TRelationGraph graph = Lambda(mt, n, probability); auto result = BenchmarkShuffleEliminationOnTopology(config, session, graph); if (!result) { @@ -350,10 +353,17 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { auto config = GetBenchmarkConfig(args); DumpBenchmarkConfig(Cout, config); + uint64_t state = 0; + if (args.HasArg("seed")) { + state = args.GetArg("seed").GetValue(); + } + BenchmarkShuffleEliminationOnTopologies( config, /*maxNumTables=*/args.GetArg("N"), - /*probabilityStep=*/args.GetArg("P")); + /*probabilityStep=*/args.GetArg("P"), + state + ); } } From 9c8f95c2b2ca307e5059a35ad73284464849e5fe Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Thu, 30 Oct 2025 04:57:37 +0000 Subject: [PATCH 19/43] Use Pitman-Yor process for key selection --- .../ut/join/kqp_join_topology_generator.cpp | 127 +++++++++++++----- .../kqp/ut/join/kqp_join_topology_generator.h | 33 ++--- ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp | 38 ++++-- 3 files changed, 135 insertions(+), 63 deletions(-) diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp index ed2e88460881..cccfafba5efb 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp @@ -1,9 +1,11 @@ #include "kqp_join_topology_generator.h" +#include #include #include #include #include +#include namespace NKikimr::NKqp { @@ -65,30 +67,92 @@ static std::string getTablePath(unsigned tableID) { return "/Root/" + getTableName(tableID); } - -static unsigned getRandom(TRNG &mt, unsigned a, unsigned b) { - std::uniform_int_distribution<> distribution(a, b); +static double getRandomNormalizedDouble(TRNG &mt) { + std::uniform_real_distribution<> distribution(0.0, 1.0); return distribution(mt); } -static unsigned getRandomBool(TRNG &mt, double trueProbability) { - std::uniform_real_distribution<> distribution(0.0, 1.0); - return distribution(mt) < trueProbability; -} +static std::vector GetPitmanYor(TRNG &mt, ui32 sum, TPitmanYorConfig config) { + std::vector keyCounts{/*initial key=*/1}; + for (ui32 column = 1; column < sum; ++ column) { + double random = getRandomNormalizedDouble(mt); -unsigned TTable::GetRandomOrNewColumn(TRNG &mt, double newColumnProbability) { - bool generateNewColumn = getRandomBool(mt, newColumnProbability); - if (generateNewColumn || NumColumns == 0) { - return NumColumns ++; + double cumulative = 0.0; + int chosenTableIndex = -1; + + for (ui32 i = 0; i < keyCounts.size(); ++ i) { + cumulative += (keyCounts[i] - config.Alpha) / (column + config.Theta); + if (random < cumulative) { + chosenTableIndex = i; + break; + } + } + + if (chosenTableIndex == -1) { + keyCounts.push_back(1); + } else { + ++ keyCounts[chosenTableIndex]; + } } - return GetRandomColumn(mt); + std::sort(keyCounts.begin(), keyCounts.end(), std::greater{}); + return keyCounts; } -unsigned TTable::GetRandomColumn(TRNG &mt) const { - assert(NumColumns != 0); - return getRandom(mt, 0, NumColumns - 1); +void TRelationGraph::SetupKeysPitmanYor(TRNG &mt, TPitmanYorConfig config) { + std::vector, /*last index*/ui32>> distributions(GetN()); + for (ui32 i = 0; i < GetN(); ++ i) { + auto distribution = GetPitmanYor(mt, AdjacencyList_[i].size(), config); + Schema_[i] = TTable{static_cast(distribution.size())}; + distributions[i] = {std::move(distribution), /*initial index=*/0}; + } + + auto GetKey = [&](ui32 node) { + auto &[nodeDistribution, lastIndex] = distributions[node]; + ui32 key = lastIndex; + + assert(lastIndex < nodeDistribution.size()); + assert(nodeDistribution[lastIndex] != 0); + if (-- nodeDistribution[lastIndex] == 0) { + ++ lastIndex; + } + + return key; + }; + + std::set> visited; + + for (ui32 u = 0; u < GetN(); ++ u) { + for (ui32 i = 0; i < AdjacencyList_[u].size(); ++ i) { + TEdge* forwardEdge = &AdjacencyList_[u][i]; + ui32 v = forwardEdge->Target; + + if (visited.contains({v, u}) || visited.contains({u, v})) { + continue; + } + + visited.insert({u, v}); + + // find backward edge + TEdge* backwardEdge = nullptr; + std::vector& targetAdjacency = AdjacencyList_[forwardEdge->Target]; + for (ui32 k = 0; k < targetAdjacency.size(); ++ k) { + if (targetAdjacency[k].Target == u) { + backwardEdge = &targetAdjacency[k]; + } + } + + ui32 ColumnLHS = GetKey(u); + ui32 ColumnRHS = GetKey(v); + + forwardEdge->ColumnLHS = ColumnLHS; + forwardEdge->ColumnRHS = ColumnRHS; + backwardEdge->ColumnLHS = ColumnRHS; + backwardEdge->ColumnRHS = ColumnLHS; + } + } + } @@ -132,7 +196,7 @@ void TSchema::Rename(std::vector oldToNew) { } -TRelationGraph TRelationGraph::FromPrufer(TRNG &mt, const std::vector& prufer, double newColumnProbability) { +TRelationGraph TRelationGraph::FromPrufer(const std::vector& prufer) { unsigned n = prufer.size() + 2; std::vector degree(n, 1); @@ -145,7 +209,7 @@ TRelationGraph TRelationGraph::FromPrufer(TRNG &mt, const std::vector& for (unsigned u : prufer) { for (unsigned v = 0; v < n; ++ v) { if (degree[v] == 1) { - graph.Connect(mt, u, v, newColumnProbability); + graph.Connect(u, v); -- degree[v]; -- degree[u]; @@ -159,7 +223,7 @@ TRelationGraph TRelationGraph::FromPrufer(TRNG &mt, const std::vector& for (; v < n; ++ v) { if (degree[v] == 1) { if (u != -1) { - graph.Connect(mt, u, v, newColumnProbability); + graph.Connect(u, v); break; } @@ -170,12 +234,9 @@ TRelationGraph TRelationGraph::FromPrufer(TRNG &mt, const std::vector& return graph; } -void TRelationGraph::Connect(TRNG &mt, unsigned lhs, unsigned rhs, double newColumnProbability) { - unsigned ColumnLHS = Schema_[lhs].GetRandomOrNewColumn(mt, newColumnProbability); - unsigned ColumnRHS = Schema_[rhs].GetRandomOrNewColumn(mt, newColumnProbability); - - AdjacencyList_[lhs].push_back({rhs, ColumnLHS, ColumnRHS}); - AdjacencyList_[rhs].push_back({lhs, ColumnRHS, ColumnLHS}); +void TRelationGraph::Connect(unsigned lhs, unsigned rhs) { + AdjacencyList_[lhs].push_back({/*Target=*/rhs, 0, 0}); + AdjacencyList_[rhs].push_back({/*Target=*/lhs, 0, 0}); } std::string TRelationGraph::MakeQuery() const { @@ -332,14 +393,14 @@ std::string TSchemaStats::ToJSON() const { return ss.str(); } -TRelationGraph GenerateLine(TRNG &mt, unsigned numNodes, double newColumnProbability) { +TRelationGraph GenerateLine(unsigned numNodes) { TRelationGraph graph(numNodes); unsigned lastVertex = 0; bool first = true; for (unsigned i = 0; i < numNodes; ++ i) { if (!first) { - graph.Connect(mt, lastVertex, i, newColumnProbability); + graph.Connect(lastVertex, i); } first = false; @@ -349,25 +410,23 @@ TRelationGraph GenerateLine(TRNG &mt, unsigned numNodes, double newColumnProbabi return graph; } -TRelationGraph GenerateStar(TRNG &mt, unsigned numNodes, double newColumnProbability) { +TRelationGraph GenerateStar(unsigned numNodes) { TRelationGraph graph(numNodes); unsigned root = 0; for (unsigned i = 1; i < numNodes; ++ i) { - graph.Connect(mt, root, i, newColumnProbability); + graph.Connect(root, i); } return graph; } -TRelationGraph GenerateFullyConnected(TRNG &mt, unsigned numNodes, - double newColumnProbability) { - +TRelationGraph GenerateFullyConnected(unsigned numNodes) { TRelationGraph graph(numNodes); for (unsigned i = 0; i < numNodes; ++ i) { for (unsigned j = i + 1; j < numNodes; ++ j) { - graph.Connect(mt, i, j, newColumnProbability); + graph.Connect(i, j); } } @@ -386,9 +445,9 @@ static std::vector GenerateRandomPruferSequence(TRNG &mt, unsigned num return prufer; } -TRelationGraph GenerateRandomTree(TRNG &mt, unsigned numNodes, double newColumnProbability) { +TRelationGraph GenerateRandomTree(TRNG &mt, unsigned numNodes) { auto prufer = GenerateRandomPruferSequence(mt, numNodes); - return TRelationGraph::FromPrufer(mt, prufer, newColumnProbability); + return TRelationGraph::FromPrufer(prufer); } } diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_generator.h b/ydb/core/kqp/ut/join/kqp_join_topology_generator.h index f89e9ec0ac55..14b6dc0da140 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_generator.h +++ b/ydb/core/kqp/ut/join/kqp_join_topology_generator.h @@ -14,25 +14,26 @@ namespace NKikimr::NKqp { using TRNG = TSerializableMT19937; + +struct TPitmanYorConfig { + double Alpha; + double Theta; +}; + + class TTable { public: TTable(unsigned numColumns = 0) - : NumColumns(numColumns) + : NumColumns_(numColumns) { } - unsigned GetRandomOrNewColumn(TRNG &mt, double newColumnProbability); - unsigned GetNumColumns() const { - return NumColumns; + return NumColumns_; } private: - unsigned GetRandomColumn(TRNG &mt) const; - -private: - // table has columns from 0..NumColumns - unsigned NumColumns; + unsigned NumColumns_; }; @@ -76,9 +77,9 @@ class TRelationGraph { { } - static TRelationGraph FromPrufer(TRNG &mt, const std::vector& prufer, double newColumnProbability); + static TRelationGraph FromPrufer(const std::vector& prufer); - void Connect(TRNG &mt, unsigned lhs, unsigned rhs, double newColumnProbability); + void Connect(unsigned lhs, unsigned rhs); std::string MakeQuery() const; @@ -98,6 +99,8 @@ class TRelationGraph { void Rename(const std::vector &oldToNew); + void SetupKeysPitmanYor(TRNG &mt, TPitmanYorConfig config); + private: struct TEdge { @@ -134,10 +137,10 @@ class TSchemaStats { }; -TRelationGraph GenerateLine(TRNG &mt, unsigned numNodes, double newColumnProbability); -TRelationGraph GenerateStar(TRNG &mt, unsigned numNodes, double newColumnProbability); -TRelationGraph GenerateFullyConnected(TRNG &mt, unsigned numNodes, double newColumnProbability); -TRelationGraph GenerateRandomTree(TRNG &mt, unsigned numNodes, double newColumnProbability); +TRelationGraph GenerateLine(unsigned numNodes); +TRelationGraph GenerateStar(unsigned numNodes); +TRelationGraph GenerateFullyConnected(unsigned numNodes); +TRelationGraph GenerateRandomTree(TRNG &mt, unsigned numNodes); } // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp index de4062290f22..569614822fa8 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp @@ -243,11 +243,16 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { } template - void BenchmarkShuffleEliminationOnTopologies(TBenchmarkConfig config, TArgs::TRangedValue numTables, TArgs::TRangedValue sameColumnProbability, uint64_t state = 0) { + void BenchmarkShuffleEliminationOnTopologies(TBenchmarkConfig config, + TArgs::TRangedValue thetaRanged, + TArgs::TRangedValue alphaRanged, + TArgs::TRangedValue numTablesRanged, + uint64_t state = 0) { + TRNG mt = TRNG::Deserialize(state); mt.reset(); - TSchema fullSchema = TSchema::MakeWithEnoughColumns(numTables.GetLast()); + TSchema fullSchema = TSchema::MakeWithEnoughColumns(numTablesRanged.GetLast()); TString stats = TSchemaStats::MakeRandom(mt, fullSchema, 7, 10).ToJSON(); mt.Restore(state); @@ -259,18 +264,22 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { auto OS = TUnbufferedFileOutput("results.csv"); DumpBoxPlotCSVHeader(OS); - for (ui64 n : numTables) { - for (double probability : sameColumnProbability) { - Cout << "\n\n"; - Cout << "================================ REPRODUCE -------------------------------\n"; - Cout << "KQP_TOPOLOGY='N=" << n << "; P=" << probability << "; seed=" << mt.Serialize() << "'\n"; - TRelationGraph graph = Lambda(mt, n, probability); - auto result = BenchmarkShuffleEliminationOnTopology(config, session, graph); - if (!result) { - goto stop; + for (double alpha : alphaRanged) { + for (double theta : thetaRanged) { + for (ui64 numTables : numTablesRanged) { + Cout << "\n\n"; + Cout << "================================ REPRODUCE -------------------------------\n"; + Cout << "KQP_TOPOLOGY='N=" << numTables << "; theta=" << theta << "; alpha=" << alpha << "; seed=" << mt.Serialize() << "'\n"; + TRelationGraph graph = Lambda(mt, numTables); + graph.SetupKeysPitmanYor(mt, TPitmanYorConfig{.Alpha = alpha, .Theta = theta}); + + auto result = BenchmarkShuffleEliminationOnTopology(config, session, graph); + if (!result) { + goto stop; + } + + DumpBoxPlotToCSV(OS, numTables, result->GetStatistics()); } - - DumpBoxPlotToCSV(OS, n, result->GetStatistics()); } } stop:; @@ -360,8 +369,9 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { BenchmarkShuffleEliminationOnTopologies( config, + /*theta=*/args.GetArg("theta"), + /*alpha=*/args.GetArg("alpha"), /*maxNumTables=*/args.GetArg("N"), - /*probabilityStep=*/args.GetArg("P"), state ); } From ccd6c9d9c7eb2e8fe70592cfba47d0ee992e10b7 Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Thu, 30 Oct 2025 05:26:12 +0000 Subject: [PATCH 20/43] Replace ForceShuffleElimination with more flexible cutoff parameter --- ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp index 569614822fa8..6b0bfcd9108b 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp @@ -62,7 +62,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { queryWithShuffleElimination += enableShuffleElimination ? "true" : "false"; queryWithShuffleElimination += "\";\n"; queryWithShuffleElimination += "PRAGMA ydb.MaxDPHypDPTableSize='4294967295';\n"; - queryWithShuffleElimination += "PRAGMA ydb.ForceShuffleElimination='true';\n"; + queryWithShuffleElimination += "PRAGMA ydb.ShuffleEliminationJoinNumCutoff='" + std::to_string(UINT32_MAX) + "';\n"; queryWithShuffleElimination += "PRAGMA ydb.CostBasedOptimizationLevel=\"" + std::to_string(optLevel) + "\";\n"; queryWithShuffleElimination += query; From 2e2066fa394651ad84bd7a585aa7d4ecda091c7d Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Thu, 30 Oct 2025 05:31:01 +0000 Subject: [PATCH 21/43] Clearer error message if arg is not provided --- ydb/core/kqp/ut/common/kqp_arg_parser.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ydb/core/kqp/ut/common/kqp_arg_parser.h b/ydb/core/kqp/ut/common/kqp_arg_parser.h index d6047f88df7e..c3605cc1e852 100644 --- a/ydb/core/kqp/ut/common/kqp_arg_parser.h +++ b/ydb/core/kqp/ut/common/kqp_arg_parser.h @@ -8,6 +8,7 @@ #include #include #include +#include namespace NKikimr::NKqp { @@ -113,6 +114,9 @@ class TArgs { template auto GetArg(std::string key) { + if (!HasArg(key)) { + throw std::out_of_range("arg not provided: '" + key + "'"); + } return ParseRangedValue(Values_[key]); } From bb4ed5430d6f45417eda625b17f84183505017f2 Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Thu, 30 Oct 2025 07:30:57 +0000 Subject: [PATCH 22/43] Implement Chung-Lu random graph model --- .../ut/join/kqp_join_topology_generator.cpp | 68 +++++++++++++ .../kqp/ut/join/kqp_join_topology_generator.h | 13 ++- ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp | 98 ++++++++++--------- 3 files changed, 134 insertions(+), 45 deletions(-) diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp index cccfafba5efb..c81f0af47498 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp @@ -450,4 +450,72 @@ TRelationGraph GenerateRandomTree(TRNG &mt, unsigned numNodes) { return TRelationGraph::FromPrufer(prufer); } +void NormalizeProbabilities(std::vector& probabilities) { + double sum = std::accumulate(probabilities.begin(), probabilities.end(), 0.0); + if (sum > 0.0) { + for (double& probability : probabilities) { + probability /= sum; + } + } +} + +std::vector SampleFromPMF(const std::vector& probabilities, + int numVertices, int minDegree) { + std::random_device randomDevice; + std::mt19937 generator(randomDevice()); + std::discrete_distribution distribution(probabilities.begin(), + probabilities.end()); + + std::vector degrees(numVertices); + for (int i = 0; i < numVertices; i++) { + degrees[i] = distribution(generator) + minDegree; + } + return degrees; +} + +std::vector GenerateLogNormalDegrees(int numVertices, double logMean, + double logStdDev, int minDegree, + int maxDegree) { + if (maxDegree == -1) maxDegree = numVertices - 1; + + std::vector probabilities(maxDegree - minDegree + 1); + for (int k = minDegree; k <= maxDegree; k++) { + if (k <= 0) { + probabilities[k - minDegree] = 0.0; + continue; + } + + double x = (double)k; + double logX = std::log(x); + double z = (logX - logMean) / logStdDev; + + // PDF: (1/(x*σ*√(2π))) * exp(-(ln(x)-μ)²/(2σ²)) + probabilities[k - minDegree] = (1.0 / (x * logStdDev * std::sqrt(2.0 * M_PI))) * + std::exp(-0.5 * z * z); + } + + NormalizeProbabilities(probabilities); + return SampleFromPMF(probabilities, numVertices, minDegree); +} + +TRelationGraph GenerateRandomChungLuGraph(TRNG &mt, const std::vector& degrees) { + TRelationGraph graph(degrees.size()); + + double sum = std::accumulate(degrees.begin(), degrees.end(), 0.0); + if (sum == 0) { + return graph; + } + + std::uniform_real_distribution<> distribution(0, 1); + for (ui32 i = 0; i < degrees.size(); ++ i) { + for (ui32 j = i + 1; j < degrees.size(); ++ j) { + if (distribution(mt) < std::min(1.0, degrees[i] * degrees[j] / sum)) { + graph.Connect(i, j); + } + } + } + + return graph; +} + } diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_generator.h b/ydb/core/kqp/ut/join/kqp_join_topology_generator.h index 14b6dc0da140..97d32ca30cbd 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_generator.h +++ b/ydb/core/kqp/ut/join/kqp_join_topology_generator.h @@ -137,10 +137,21 @@ class TSchemaStats { }; + + +void NormalizeProbabilities(std::vector& probabilities); + +std::vector SampleFromPMF(const std::vector& probabilities, + int numVertices, int minDegree); + +std::vector GenerateLogNormalDegrees(int numVertices, double logMean = 1.0, + double logStdDev = 0.5, int minDegree = 1, + int maxDegree = -1); TRelationGraph GenerateLine(unsigned numNodes); TRelationGraph GenerateStar(unsigned numNodes); TRelationGraph GenerateFullyConnected(unsigned numNodes); TRelationGraph GenerateRandomTree(TRNG &mt, unsigned numNodes); -} // namespace NKikimr::NKqp +TRelationGraph GenerateRandomChungLuGraph(TRNG &mt, const std::vector& degrees); +} // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp index 6b0bfcd9108b..5d04bae145f1 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp @@ -242,12 +242,13 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { OS << "}\n"; } - template - void BenchmarkShuffleEliminationOnTopologies(TBenchmarkConfig config, - TArgs::TRangedValue thetaRanged, - TArgs::TRangedValue alphaRanged, - TArgs::TRangedValue numTablesRanged, - uint64_t state = 0) { + template + void BenchmarkShuffleEliminationOnTopologies(TBenchmarkConfig config, + TArgs::TRangedValue thetaRanged, + TArgs::TRangedValue alphaRanged, + TArgs::TRangedValue numTablesRanged, + uint64_t state, + Lambda &&lambda) { TRNG mt = TRNG::Deserialize(state); @@ -269,8 +270,9 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { for (ui64 numTables : numTablesRanged) { Cout << "\n\n"; Cout << "================================ REPRODUCE -------------------------------\n"; - Cout << "KQP_TOPOLOGY='N=" << numTables << "; theta=" << theta << "; alpha=" << alpha << "; seed=" << mt.Serialize() << "'\n"; - TRelationGraph graph = Lambda(mt, numTables); + Cout << "TOPOLOGY='N=" << numTables << "; theta=" << theta << "; alpha=" << alpha << "; seed=" << mt.Serialize() << "'\n"; + + TRelationGraph graph = lambda(mt, numTables); graph.SetupKeysPitmanYor(mt, TPitmanYorConfig{.Alpha = alpha, .Theta = theta}); auto result = BenchmarkShuffleEliminationOnTopology(config, session, graph); @@ -285,6 +287,50 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { stop:; } + // Y_UNIT_TEST(ShuffleEliminationBenchmarkOnStar) { + // BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/15, /*probabilityStep=*/0.2); + // } + + // Y_UNIT_TEST(ShuffleEliminationBenchmarkOnLine) { + // BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/15, /*probabilityStep=*/0.2); + // } + + // Y_UNIT_TEST(ShuffleEliminationBenchmarkOnFullyConnected) { + // BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/15, /*probabilityStep=*/0.2); + // } + + // Y_UNIT_TEST(ShuffleEliminationBenchmarkOnRandomTree) { + // BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/25, /*probabilityStep=*/0.2); + // } + + + + Y_UNIT_TEST(Benchmark) { + TArgs args{GetTestParam("TOPOLOGY")}; + + auto config = GetBenchmarkConfig(args); + DumpBenchmarkConfig(Cout, config); + + uint64_t state = 0; + if (args.HasArg("seed")) { + state = args.GetArg("seed").GetValue(); + } + + + auto GenerateTopology = [](TRNG &mt, uint64_t n) { + return GenerateRandomChungLuGraph(mt, GenerateLogNormalDegrees(n)); + }; + + BenchmarkShuffleEliminationOnTopologies( + config, + /*theta=*/args.GetArg("theta"), + /*alpha=*/args.GetArg("alpha"), + /*maxNumTables=*/args.GetArg("N"), + state, + GenerateTopology + ); + } + Y_UNIT_TEST(TRunningStatistics) { TRunningStatistics stats; @@ -340,42 +386,6 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { UNIT_ASSERT_EQUAL(stats.GetN(), 10); } - // Y_UNIT_TEST(ShuffleEliminationBenchmarkOnStar) { - // BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/15, /*probabilityStep=*/0.2); - // } - - // Y_UNIT_TEST(ShuffleEliminationBenchmarkOnLine) { - // BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/15, /*probabilityStep=*/0.2); - // } - - // Y_UNIT_TEST(ShuffleEliminationBenchmarkOnFullyConnected) { - // BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/15, /*probabilityStep=*/0.2); - // } - - // Y_UNIT_TEST(ShuffleEliminationBenchmarkOnRandomTree) { - // BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/25, /*probabilityStep=*/0.2); - // } - - Y_UNIT_TEST(Benchmark) { - TArgs args{GetTestParam("TOPOLOGY")}; - - auto config = GetBenchmarkConfig(args); - DumpBenchmarkConfig(Cout, config); - - uint64_t state = 0; - if (args.HasArg("seed")) { - state = args.GetArg("seed").GetValue(); - } - - BenchmarkShuffleEliminationOnTopologies( - config, - /*theta=*/args.GetArg("theta"), - /*alpha=*/args.GetArg("alpha"), - /*maxNumTables=*/args.GetArg("N"), - state - ); - } - } } From e58421b59503f4631cc8a1cf566044a768ca5503 Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Thu, 30 Oct 2025 09:13:15 +0000 Subject: [PATCH 23/43] Implement fixed degree topology with HavalHakimi + MCMC + MH --- .../ut/join/kqp_join_topology_generator.cpp | 199 ++++++++++++++++++ .../kqp/ut/join/kqp_join_topology_generator.h | 90 +++++++- ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp | 10 +- 3 files changed, 292 insertions(+), 7 deletions(-) diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp index c81f0af47498..c61b4b215ff3 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp @@ -518,4 +518,203 @@ TRelationGraph GenerateRandomChungLuGraph(TRNG &mt, const std::vector& degr return graph; } + +void MakeEvenSum(std::vector& degrees) { + int sum = std::accumulate(degrees.begin(), degrees.end(), 0); + if (sum % 2 == 1) { + auto minIt = std::min_element(degrees.begin(), degrees.end()); + ++ *minIt; + } +} + +bool SatisfiesErdosGallai(std::vector degrees) { + int sum = std::accumulate(degrees.begin(), degrees.end(), 0); + + if (sum % 2 != 0) { + return false; + } + + for (int degree : degrees) { + if (degree < 0 || degree >= static_cast(degrees.size())) { + return false; + } + } + + std::sort(degrees.rbegin(), degrees.rend()); + + uint64_t sumLeft = 0; + for (uint64_t k = 0; k < degrees.size(); ++ k) { + sumLeft += degrees[k]; + + uint64_t sumRight = k * (k + 1); + for (uint64_t i = k + 1; i < degrees.size(); ++ i) { + sumRight += std::min(k + 1, degrees[i]); + } + + if (sumLeft > sumRight) { + return false; + } + } + + return true; +} + +bool CanBeConnected(const std::vector& degrees) { + int n = degrees.size(); + if (n <= 1) { + return true; + } + + int sum = std::accumulate(degrees.begin(), degrees.end(), 0); + if (sum < 2 * (n - 1)) { + return false; + } + + for (int degree : degrees) { + if (degree == 0) { + return false; + } + } + + return true; +} + +std::vector MakeGraphicConnected(std::vector degrees) { + const i32 MAX_ITERATIONS = 1000; + + for (int& degree : degrees) { + if (degree == 0) { + degree = 1; + } + } + + // Cap degrees at n-1 + for (int& degree : degrees) { + degree = std::min(degree, degrees.size() - 1); + } + + MakeEvenSum(degrees); + + int iterations = 0; + while ((!SatisfiesErdosGallai(degrees) || !CanBeConnected(degrees)) && iterations ++ < MAX_ITERATIONS) { + std::sort(degrees.begin(), degrees.end(), std::greater()); + + if (!SatisfiesErdosGallai(degrees)) { + // Reduce max, increase min (redistribute) + if (degrees[0] > degrees[degrees.size() - 1] + 1) { + -- degrees[0]; + ++ degrees[degrees.size() - 1]; + } else { + -- degrees[0]; + } + } else if (!CanBeConnected(degrees)) { + // Increase minimum degrees to help connectivity + for (int& degree : degrees) { + ui32 edges = std::accumulate(degrees.begin(), degrees.end(), 0) / 2; + if (degree < 2 && edges + 2 <= degrees.size() * (degrees.size() - 1) / 2) { + ++ degree; + } + } + } + + MakeEvenSum(degrees); + } + + return degrees; +} + +TRelationGraph ConstructGraphHavelHakimi(std::vector degrees) { + TRelationGraph graph(degrees.size()); + + std::vector> nodes; + for (uint32_t i = 0; i < degrees.size(); ++i) { + nodes.push_back({degrees[i], i}); + } + + while (true) { + std::sort(nodes.begin(), nodes.end(), std::greater>{}); + + while (!nodes.empty() && nodes.back().first == 0) { + nodes.pop_back(); + } + + if (nodes.empty()) { + break; + } + + auto [degree, u] = nodes[0]; + nodes.erase(nodes.begin()); + + if (degree > static_cast(nodes.size())) { + break; + } + + for (uint32_t i = 0; i < static_cast(degree); ++ i) { + uint32_t v = nodes[i].second; + graph.Connect(u, v); + -- nodes[i].first; + } + } + + return graph; +} + +void MCMCRandomize(TRNG &mt, TRelationGraph& graph, int numSwaps) { + std::uniform_int_distribution<> nodeDist(0, graph.GetN() - 1); + std::uniform_real_distribution<> probDist(0.0, 1.0); + + auto &adjacency = graph.GetAdjacencyList(); + + const double TEMP_START = 5.0; + const double TEMP_END = 0.1; + const double CONNECTIVITY_PENALTY = 20.0; + const int MAX_ATTEMPTS = numSwaps * 10; + + int successfulSwaps = 0; + int attempt = 0; + + while (attempt < MAX_ATTEMPTS && (successfulSwaps < numSwaps || !graph.IsConnected())) { + double progress = std::min(1.0, static_cast(attempt) / MAX_ATTEMPTS); + double temperature = TEMP_START * std::pow(TEMP_END / TEMP_START, progress); + + ++ attempt; + + int a = nodeDist(mt); + if (adjacency[a].empty()) continue; + + int bIdx = std::uniform_int_distribution<>(0, adjacency[a].size() - 1)(mt); + int b = adjacency[a][bIdx].Target; + + int c = nodeDist(mt); + if (c == a || c == b || adjacency[c].empty()) continue; + + int dIdx = std::uniform_int_distribution<>(0, adjacency[c].size() - 1)(mt); + int d = adjacency[c][dIdx].Target; + + if (d == a || d == b || d == c) continue; + if (graph.HasEdge(a, c) || graph.HasEdge(b, d)) continue; + + int oldComponents = graph.NumComponents(); + + graph.Disconnect(a, b); + graph.Disconnect(c, d); + graph.Connect(a, c); + graph.Connect(b, d); + + int newComponents = graph.NumComponents(); + double deltaEnergy = CONNECTIVITY_PENALTY * (newComponents - oldComponents); + + bool accept = probDist(mt) < std::exp(-deltaEnergy / temperature); + + if (accept) { + ++ successfulSwaps; + } else { + graph.Disconnect(a, c); + graph.Disconnect(b, d); + graph.Connect(a, b); + graph.Connect(c, d); + } + } +} + } diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_generator.h b/ydb/core/kqp/ut/join/kqp_join_topology_generator.h index 97d32ca30cbd..18d17db2bd24 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_generator.h +++ b/ydb/core/kqp/ut/join/kqp_join_topology_generator.h @@ -7,6 +7,7 @@ #include #include #include +#include namespace NKikimr::NKqp { @@ -81,6 +82,73 @@ class TRelationGraph { void Connect(unsigned lhs, unsigned rhs); + void Disconnect(unsigned u, unsigned v) { + auto& adjacencyU = AdjacencyList_[u]; + auto& adjacencyV = AdjacencyList_[v]; + adjacencyU.erase(std::remove_if(adjacencyU.begin(), adjacencyU.end(), [v](TEdge edge) { return edge.Target == v; }), adjacencyU.end()); + adjacencyV.erase(std::remove_if(adjacencyV.begin(), adjacencyV.end(), [u](TEdge edge) { return edge.Target == u; }), adjacencyV.end()); + + } + + bool HasEdge(unsigned u, unsigned v) const { + for (ui32 i = 0; i < AdjacencyList_[u].size(); ++i) { + if (AdjacencyList_[u][i].Target == v) { + return true; + } + } + + return false; + } + + std::vector FindComponents() const { + std::vector component(GetN(), -1); + int numComponents = 0; + + for (unsigned start = 0; start < GetN(); ++ start) { + if (component[start] != -1) { + continue; + } + + std::queue queue; + queue.push(start); + component[start] = numComponents; + + while (!queue.empty()) { + unsigned u = queue.front(); + queue.pop(); + for (TEdge edge : AdjacencyList_[u]) { + unsigned v = edge.Target; + if (component[v] == -1) { + component[v] = numComponents; + queue.push(v); + } + } + } + + ++ numComponents; + } + + return component; + } + + int NumComponents() const { + auto comp = FindComponents(); + return comp.empty() ? 0 : *std::max_element(comp.begin(), comp.end()) + 1; + } + + bool IsConnected() const { + return NumComponents() == 1; + } + + int NumEdges() const { + int count = 0; + for (const auto& adjacency : AdjacencyList_) { + count += adjacency.size(); + } + return count / 2; + } + + std::string MakeQuery() const; void DumpGraph(IOutputStream &OS) const; @@ -101,8 +169,7 @@ class TRelationGraph { void SetupKeysPitmanYor(TRNG &mt, TPitmanYorConfig config); - -private: +public: struct TEdge { unsigned Target; unsigned ColumnLHS, ColumnRHS; @@ -110,6 +177,12 @@ class TRelationGraph { using TAdjacencyList = std::vector>; +public: + TAdjacencyList& GetAdjacencyList() { + return AdjacencyList_; + } + +private: TAdjacencyList AdjacencyList_; TSchema Schema_; }; @@ -136,9 +209,6 @@ class TSchemaStats { std::vector Stats_; }; - - - void NormalizeProbabilities(std::vector& probabilities); std::vector SampleFromPMF(const std::vector& probabilities, @@ -147,6 +217,16 @@ std::vector SampleFromPMF(const std::vector& probabilities, std::vector GenerateLogNormalDegrees(int numVertices, double logMean = 1.0, double logStdDev = 0.5, int minDegree = 1, int maxDegree = -1); + +TRelationGraph ConstructGraphHavelHakimi(std::vector degrees); + +std::vector MakeGraphicConnected(std::vector degrees); + +void MCMCRandomize(TRNG &mt, TRelationGraph& graph, int numSwaps); + + + + TRelationGraph GenerateLine(unsigned numNodes); TRelationGraph GenerateStar(unsigned numNodes); TRelationGraph GenerateFullyConnected(unsigned numNodes); diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp index 5d04bae145f1..f372929a254d 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp @@ -316,9 +316,15 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { state = args.GetArg("seed").GetValue(); } + auto initialDegrees = GenerateLogNormalDegrees(15); + auto fixedDegrees = MakeGraphicConnected(initialDegrees); + auto graph = ConstructGraphHavelHakimi(fixedDegrees); - auto GenerateTopology = [](TRNG &mt, uint64_t n) { - return GenerateRandomChungLuGraph(mt, GenerateLogNormalDegrees(n)); + auto GenerateTopology = [&](TRNG &mt, uint64_t n) { + (void) n; + MCMCRandomize(mt, graph, 50); + + return graph; }; BenchmarkShuffleEliminationOnTopologies( From 38d00a939a4035258eda052177ee99ad0fffbc31 Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Thu, 30 Oct 2025 14:05:35 +0000 Subject: [PATCH 24/43] Add convenient launcher for many topologies --- ydb/core/kqp/ut/common/kqp_arg_parser.h | 18 +- ydb/core/kqp/ut/common/kqp_benches.h | 6 +- .../ut/join/kqp_join_topology_generator.cpp | 6 +- .../kqp/ut/join/kqp_join_topology_generator.h | 30 ++- ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp | 233 +++++++++++++----- 5 files changed, 214 insertions(+), 79 deletions(-) diff --git a/ydb/core/kqp/ut/common/kqp_arg_parser.h b/ydb/core/kqp/ut/common/kqp_arg_parser.h index c3605cc1e852..9fe4be03630b 100644 --- a/ydb/core/kqp/ut/common/kqp_arg_parser.h +++ b/ydb/core/kqp/ut/common/kqp_arg_parser.h @@ -112,12 +112,24 @@ class TArgs { { } - template - auto GetArg(std::string key) { + std::string GetString(std::string key) { if (!HasArg(key)) { throw std::out_of_range("arg not provided: '" + key + "'"); } - return ParseRangedValue(Values_[key]); + return Values_[key]; + } + + template + auto GetArg(std::string key) { + return ParseRangedValue(GetString(key)); + } + + template + auto GetArgOrDefault(std::string key, std::string defaultSerialized) { + if (HasArg(key)) { + return GetArg(key); + } + return ParseRangedValue(defaultSerialized); } bool HasArg(std::string key) { diff --git a/ydb/core/kqp/ut/common/kqp_benches.h b/ydb/core/kqp/ut/common/kqp_benches.h index c2b314ef8bb7..8ea2700f3fe3 100644 --- a/ydb/core/kqp/ut/common/kqp_benches.h +++ b/ydb/core/kqp/ut/common/kqp_benches.h @@ -394,12 +394,12 @@ static std::string joinVector(const std::vector &data) { void DumpBoxPlotCSVHeader(IOutputStream &OS) { - OS << "N,median,Q1,Q3,IQR,MAD,outliers\n"; + OS << "median,Q1,Q3,IQR,MAD,min,max"; } template -void DumpBoxPlotToCSV(IOutputStream &OS, ui32 i, TStatistics stat) { - OS << i << "," << stat.Median << "," << stat.Q1 << "," << stat.Q3 << "," << stat.IQR << "," << stat.MAD << "," << joinVector(stat.Outliers) << "\n"; +void DumpBoxPlotToCSV(IOutputStream &OS, TStatistics stat) { + OS << stat.Median << "," << stat.Q1 << "," << stat.Q3 << "," << stat.IQR << "," << stat.MAD << "," << stat.Min << "," << stat.Max; } diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp index c61b4b215ff3..485f73245623 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp @@ -393,7 +393,7 @@ std::string TSchemaStats::ToJSON() const { return ss.str(); } -TRelationGraph GenerateLine(unsigned numNodes) { +TRelationGraph GenerateLine([[maybe_unused]] TRNG &rng, unsigned numNodes) { TRelationGraph graph(numNodes); unsigned lastVertex = 0; @@ -410,7 +410,7 @@ TRelationGraph GenerateLine(unsigned numNodes) { return graph; } -TRelationGraph GenerateStar(unsigned numNodes) { +TRelationGraph GenerateStar([[maybe_unused]] TRNG &rng, unsigned numNodes) { TRelationGraph graph(numNodes); unsigned root = 0; @@ -421,7 +421,7 @@ TRelationGraph GenerateStar(unsigned numNodes) { return graph; } -TRelationGraph GenerateFullyConnected(unsigned numNodes) { +TRelationGraph GenerateFullyConnected([[maybe_unused]] TRNG &rng, unsigned numNodes) { TRelationGraph graph(numNodes); for (unsigned i = 0; i < numNodes; ++ i) { diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_generator.h b/ydb/core/kqp/ut/join/kqp_join_topology_generator.h index 18d17db2bd24..8f25ce8dbf03 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_generator.h +++ b/ydb/core/kqp/ut/join/kqp_join_topology_generator.h @@ -19,6 +19,14 @@ using TRNG = TSerializableMT19937; struct TPitmanYorConfig { double Alpha; double Theta; + + void DumpParamsHeader(IOutputStream &OS) { + OS << "alpha,theta"; + } + + void DumpParams(IOutputStream &OS) { + OS << Alpha << "," << Theta; + } }; @@ -209,14 +217,17 @@ class TSchemaStats { std::vector Stats_; }; + void NormalizeProbabilities(std::vector& probabilities); -std::vector SampleFromPMF(const std::vector& probabilities, - int numVertices, int minDegree); +std::vector SampleFromPMF( + const std::vector& probabilities, + int numVertices, int minDegree); -std::vector GenerateLogNormalDegrees(int numVertices, double logMean = 1.0, - double logStdDev = 0.5, int minDegree = 1, - int maxDegree = -1); +std::vector GenerateLogNormalDegrees( + int numVertices, double logMean = 1.0, double logStdDev = 0.5, + int minDegree = 1, int maxDegree = -1 +); TRelationGraph ConstructGraphHavelHakimi(std::vector degrees); @@ -224,12 +235,9 @@ std::vector MakeGraphicConnected(std::vector degrees); void MCMCRandomize(TRNG &mt, TRelationGraph& graph, int numSwaps); - - - -TRelationGraph GenerateLine(unsigned numNodes); -TRelationGraph GenerateStar(unsigned numNodes); -TRelationGraph GenerateFullyConnected(unsigned numNodes); +TRelationGraph GenerateLine(TRNG &rng, unsigned numNodes); +TRelationGraph GenerateStar(TRNG &rng, unsigned numNodes); +TRelationGraph GenerateFullyConnected(TRNG &rng, unsigned numNodes); TRelationGraph GenerateRandomTree(TRNG &mt, unsigned numNodes); TRelationGraph GenerateRandomChungLuGraph(TRNG &mt, const std::vector& degrees); diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp index f372929a254d..397a5ac49485 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp @@ -1,7 +1,11 @@ +#include +#include +#include #include #include #include +#include #include #include #include @@ -132,7 +136,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { return result.WithShuffleElimination / result.WithoutShuffleElimination; } - TKikimrRunner GetCBOTestsYDB(TString stats, TDuration compilationTimeout) { + std::unique_ptr GetCBOTestsYDB(TString stats, TDuration compilationTimeout) { TVector settings; assert(!stats.empty()); @@ -151,7 +155,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { serverSettings.SetWithSampleTables(false); serverSettings.SetKqpSettings(settings); - return TKikimrRunner(serverSettings); + return std::make_unique(serverSettings); } std::optional> BenchmarkShuffleEliminationOnTopology(TBenchmarkConfig config, NYdb::NQuery::TSession session, TRelationGraph graph) { @@ -242,71 +246,190 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { OS << "}\n"; } - template - void BenchmarkShuffleEliminationOnTopologies(TBenchmarkConfig config, - TArgs::TRangedValue thetaRanged, - TArgs::TRangedValue alphaRanged, - TArgs::TRangedValue numTablesRanged, - uint64_t state, - Lambda &&lambda) { - TRNG mt = TRNG::Deserialize(state); + class TDegreeDistributionGenerator { + public: + virtual std::vector Initialize(TArgs args) = 0; + virtual std::vector GenerateDegreeSequence(TRNG &rng) = 0; - mt.reset(); + virtual ~TDegreeDistributionGenerator() = default; + }; + + class TTopologyTester { + public: + virtual void Initialize(TArgs args) = 0; + virtual TRelationGraph ProduceGraph(TRNG &rng) = 0; + virtual void DumpParamsHeader(IOutputStream &OS) = 0; + virtual void DumpParams(IOutputStream &OS) = 0; + virtual void Loop(std::function) {} + + virtual ~TTopologyTester() = default; + }; + + + TPitmanYorConfig GetPitmanYorConfig(TArgs args) { + return TPitmanYorConfig{ + .Alpha = args.GetArgOrDefault("alpha", "0.5").GetValue(), + .Theta = args.GetArgOrDefault("theta", "1.0").GetValue() + }; + } + + class TMCMCTester : public TTopologyTester { + public: + void Initialize(TArgs args) override { + N_ = args.GetArg("N").GetValue(); + + auto initialDegrees = GenerateLogNormalDegrees(N_); + auto fixedDegrees = MakeGraphicConnected(initialDegrees); + InitialGraph_ = ConstructGraphHavelHakimi(fixedDegrees); + } + + TRelationGraph ProduceGraph(TRNG &rng) override { + TRelationGraph graph = InitialGraph_; + MCMCRandomize(rng, graph, /*MCMCSteps*/100); + return graph; + } + + void DumpParamsHeader(IOutputStream &OS) override { + OS << "N,"; + } + + void DumpParams(IOutputStream &OS) override { + OS << N_ << ","; + } + + private: + TRelationGraph InitialGraph_; + ui32 N_; + }; + + struct TTestContext { + std::unique_ptr Runner; + NYdb::NQuery::TQueryClient QueryClient; + NYdb::NQuery::TSession Session; + + TRNG RNG; + std::optional OS; + }; + + TTestContext CreateTestContext(TArgs args, uint64_t state = 0, std::string outputFile = "") { + TRNG rng = TRNG::Deserialize(state); + + auto numTablesRanged = args.GetArg("N"); + + rng.reset(); // ensure this setup is always the same TSchema fullSchema = TSchema::MakeWithEnoughColumns(numTablesRanged.GetLast()); - TString stats = TSchemaStats::MakeRandom(mt, fullSchema, 7, 10).ToJSON(); + TString stats = TSchemaStats::MakeRandom(rng, fullSchema, 7, 10).ToJSON(); - mt.Restore(state); + rng.Restore(state); auto kikimr = GetCBOTestsYDB(stats, TDuration::Seconds(10)); - auto db = kikimr.GetQueryClient(); + auto db = kikimr->GetQueryClient(); auto session = db.GetSession().GetValueSync().GetSession(); - auto OS = TUnbufferedFileOutput("results.csv"); - DumpBoxPlotCSVHeader(OS); + std::optional os = std::nullopt; + if (!outputFile.empty()) { + os = TUnbufferedFileOutput(outputFile); + } - for (double alpha : alphaRanged) { - for (double theta : thetaRanged) { - for (ui64 numTables : numTablesRanged) { - Cout << "\n\n"; - Cout << "================================ REPRODUCE -------------------------------\n"; - Cout << "TOPOLOGY='N=" << numTables << "; theta=" << theta << "; alpha=" << alpha << "; seed=" << mt.Serialize() << "'\n"; + return { std::move(kikimr), std::move(db), std::move(session), std::move(rng), std::move(os) }; + } - TRelationGraph graph = lambda(mt, numTables); - graph.SetupKeysPitmanYor(mt, TPitmanYorConfig{.Alpha = alpha, .Theta = theta}); + void RunMCMC(TTestContext &ctx, TBenchmarkConfig config, TArgs args) { + if (ctx.OS) { + (*ctx.OS) << "N,alpha,theta,logStdDev,logMean,repeats,seed,"; + DumpBoxPlotCSVHeader(*ctx.OS); + (*ctx.OS) << "\n"; + } - auto result = BenchmarkShuffleEliminationOnTopology(config, session, graph); - if (!result) { - goto stop; + ui64 mcmcSteps = args.GetArgOrDefault("mcmcSteps", "100").GetValue(); + for (ui64 n : args.GetArg("N")) { + for (double alpha : args.GetArgOrDefault("alpha", "0.5")) { + for (double theta : args.GetArgOrDefault("theta", "1.0")) { + for (double logStdDev : args.GetArgOrDefault("logStdDev", "0.5")) { + for (double logMean : args.GetArgOrDefault("logMean", "1.0")) { + auto initialDegrees = GenerateLogNormalDegrees(n, logMean, logStdDev); + auto fixedDegrees = MakeGraphicConnected(initialDegrees); + auto initialGraph = ConstructGraphHavelHakimi(fixedDegrees); + + ui64 repeats = args.GetArgOrDefault("repeats", "1").GetValue(); + for (ui64 i = 0; i < repeats; ++ i) { + Cout << "Reproduce: 'N=" << n << "; alpha=" << alpha << "; theta=" + << theta << "; logStdDev=" << logStdDev << "; logMean=" << logMean + << "; seed=" << ctx.RNG.Serialize() << "'\n"; + + ui64 seed = ctx.RNG.Serialize(); + + TRelationGraph graph = initialGraph; + MCMCRandomize(ctx.RNG, graph, mcmcSteps); + graph.SetupKeysPitmanYor(ctx.RNG, TPitmanYorConfig{.Alpha = alpha, .Theta = theta}); + + auto result = BenchmarkShuffleEliminationOnTopology(config, ctx.Session, graph); + if (!result) { + goto stop; + } + + + if (ctx.OS) { + (*ctx.OS) << n << "," << alpha << "," << theta << "," << logStdDev + << "," << logMean << "," << repeats << "," << seed << ","; + DumpBoxPlotToCSV(*ctx.OS, result->GetStatistics()); + (*ctx.OS) << "\n"; + } + } + } } - - DumpBoxPlotToCSV(OS, numTables, result->GetStatistics()); } } } stop:; } - // Y_UNIT_TEST(ShuffleEliminationBenchmarkOnStar) { - // BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/15, /*probabilityStep=*/0.2); - // } + template + void RunTrivialTopology(TTestContext &ctx, TBenchmarkConfig config, TArgs args) { + if (ctx.OS) { + (*ctx.OS) << "N,alpha,theta,seed,"; + DumpBoxPlotCSVHeader(*ctx.OS); + (*ctx.OS) << "\n"; + } - // Y_UNIT_TEST(ShuffleEliminationBenchmarkOnLine) { - // BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/15, /*probabilityStep=*/0.2); - // } + for (ui64 n : args.GetArg("N")) { + for (double alpha : args.GetArgOrDefault("alpha", "0.5")) { + for (double theta : args.GetArgOrDefault("theta", "1.0")) { + ui64 seed = ctx.RNG.Serialize(); - // Y_UNIT_TEST(ShuffleEliminationBenchmarkOnFullyConnected) { - // BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/15, /*probabilityStep=*/0.2); - // } + Cout << "Reproduce: 'N=" << n << "; alpha=" << alpha << "; theta=" + << theta << "; seed=" << seed << "'\n"; - // Y_UNIT_TEST(ShuffleEliminationBenchmarkOnRandomTree) { - // BenchmarkShuffleEliminationOnTopologies(/*maxNumTables=*/25, /*probabilityStep=*/0.2); - // } + TRelationGraph graph = TrivialTopology(ctx.RNG, n); + graph.SetupKeysPitmanYor(ctx.RNG, TPitmanYorConfig{.Alpha = alpha, .Theta = theta}); + auto result = BenchmarkShuffleEliminationOnTopology(config, ctx.Session, graph); + if (!result) { + goto stop; + } + if (ctx.OS) { + (*ctx.OS) << n << "," << alpha << "," << theta << "," << seed << ","; + DumpBoxPlotToCSV(*ctx.OS, result->GetStatistics()); + (*ctx.OS) << "\n"; + } + } + } + } + stop:; + } Y_UNIT_TEST(Benchmark) { TArgs args{GetTestParam("TOPOLOGY")}; + if (!args.HasArg("N")) { + return; + } + + std::string topology = "star"; + if (args.HasArg("type")) { + topology = args.GetString("type"); + } auto config = GetBenchmarkConfig(args); DumpBenchmarkConfig(Cout, config); @@ -316,25 +439,17 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { state = args.GetArg("seed").GetValue(); } - auto initialDegrees = GenerateLogNormalDegrees(15); - auto fixedDegrees = MakeGraphicConnected(initialDegrees); - auto graph = ConstructGraphHavelHakimi(fixedDegrees); - - auto GenerateTopology = [&](TRNG &mt, uint64_t n) { - (void) n; - MCMCRandomize(mt, graph, 50); - - return graph; - }; + TTestContext ctx = CreateTestContext(args, state, GetTestParam("SAVE_FILE")); - BenchmarkShuffleEliminationOnTopologies( - config, - /*theta=*/args.GetArg("theta"), - /*alpha=*/args.GetArg("alpha"), - /*maxNumTables=*/args.GetArg("N"), - state, - GenerateTopology - ); + if (topology == "mcmc") { + RunMCMC(ctx, config, args); + } else if (topology == "star") { + RunTrivialTopology(ctx, config, args); + } else if (topology == "path") { + RunTrivialTopology(ctx, config, args); + } else if (topology == "clique") { + RunTrivialTopology(ctx, config, args); + } } From 448e7e2fac5d31421e099454ac24c7b1e7775fc1 Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Fri, 31 Oct 2025 05:16:39 +0000 Subject: [PATCH 25/43] Allow to collect different values --- ydb/core/kqp/ut/common/kqp_benches.h | 21 +++ ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp | 128 ++++++++++++------ 2 files changed, 108 insertions(+), 41 deletions(-) diff --git a/ydb/core/kqp/ut/common/kqp_benches.h b/ydb/core/kqp/ut/common/kqp_benches.h index 8ea2700f3fe3..0776e05dbfcd 100644 --- a/ydb/core/kqp/ut/common/kqp_benches.h +++ b/ydb/core/kqp/ut/common/kqp_benches.h @@ -303,6 +303,13 @@ class TRunningStatistics { }; } + void AddValues(const TRunningStatistics& other) { + TRunningStatistics values; + other.IterateElements([&](TValue value) { + AddValue(value); + }); + } + std::vector CollectElements() const { std::vector values; IterateElements([&](TValue value) { @@ -312,6 +319,20 @@ class TRunningStatistics { return values; } + template + TRunningStatistics Cast() const { + if constexpr (std::is_same_v) { + return *this; + } else { + TRunningStatistics stats; + IterateElements([&](TValue lhs) { + stats.AddValue(static_cast(lhs)); + }); + + return stats; + } + } + TRunningStatistics operator-(TRunningStatistics rhsStats) const { TRunningStatistics stats; IterateElements([&](TValue lhs) { diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp index 397a5ac49485..e1070ae6c4c7 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp @@ -98,42 +98,63 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { return stats; } + std::optional> BenchmarkShuffleElimination(TBenchmarkConfig config, NYdb::NQuery::TSession session, std::string resultType, const TString& query) { + Cout << resultType << "\n"; + + std::optional> withoutCBO; + if (resultType.contains("0")) { + Cout << "--------------------------------- W/O CBO --------------------------------\n"; + withoutCBO = BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/false, /*optLevel=*/0)); + if (!withoutCBO) { + return std::nullopt; + } - struct ShuffleEliminationBenchmarkResult { - TRunningStatistics WithoutCBO; - TRunningStatistics WithShuffleElimination; - TRunningStatistics WithoutShuffleElimination; - }; - - std::optional> BenchmarkShuffleElimination(TBenchmarkConfig config, NYdb::NQuery::TSession session, const TString& query) { - Cout << "--------------------------------- W/O CBO --------------------------------\n"; - auto withoutCBO = BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/false, /*optLevel=*/0)); - if (!withoutCBO) { - return std::nullopt; + if (resultType == "0") { + return withoutCBO->Cast(); + } } - Cout << "--------------------------------- CBO-SE ---------------------------------\n"; - auto withoutShuffleElimination = BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/false, /*optLevel=*/2)); - if (!withoutShuffleElimination) { - return std::nullopt; - } + std::optional> withoutShuffleElimination; + if (resultType.contains("CBO")) { + Cout << "--------------------------------- CBO-SE ---------------------------------\n"; + withoutShuffleElimination = BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/false, /*optLevel=*/2)); + if (!withoutShuffleElimination) { + return std::nullopt; + } - Cout << "--------------------------------- CBO+SE ---------------------------------\n"; - auto withShuffleElimination = BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/true, /*optLevel=*/2)); - if (!withShuffleElimination) { - return std::nullopt; + if (resultType == "CBO") { + return withoutShuffleElimination->Cast(); + } + + if (resultType == "CBO-0") { + return (*withoutShuffleElimination - *withoutCBO).Cast(); + } } - Cout << "--------------------------------------------------------------------------\n"; + std::optional> withShuffleElimination; + if (resultType.contains("SE")) { + Cout << "--------------------------------- CBO+SE ---------------------------------\n"; + withShuffleElimination = BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/true, /*optLevel=*/2)); + if (!withShuffleElimination) { + return std::nullopt; + } + if (resultType == "SE") { + return withShuffleElimination->Cast(); + } - ShuffleEliminationBenchmarkResult result { - .WithoutCBO = *withoutCBO, - .WithShuffleElimination = *withShuffleElimination - *withoutCBO, - .WithoutShuffleElimination = *withoutShuffleElimination - *withoutCBO - }; + if (resultType == "SE-0") { + return (*withShuffleElimination - *withoutCBO).Cast(); + } + } + + Cout << "--------------------------------------------------------------------------\n"; - return result.WithShuffleElimination / result.WithoutShuffleElimination; + if (resultType == "SE-0/CBO-0") { + return (*withShuffleElimination - *withoutCBO) / (*withoutShuffleElimination - *withoutCBO); + } else { + return *withShuffleElimination / *withoutShuffleElimination; + } } std::unique_ptr GetCBOTestsYDB(TString stats, TDuration compilationTimeout) { @@ -158,7 +179,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { return std::make_unique(serverSettings); } - std::optional> BenchmarkShuffleEliminationOnTopology(TBenchmarkConfig config, NYdb::NQuery::TSession session, TRelationGraph graph) { + std::optional> BenchmarkShuffleEliminationOnTopology(TBenchmarkConfig config, NYdb::NQuery::TSession session, std::string resultType, TRelationGraph graph) { Cout << "================================= CREATE =================================\n"; graph.DumpGraph(Cout); @@ -176,7 +197,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { Cout << "================================= BENCHMARK ==============================\n"; TString query = graph.MakeQuery(); Cout << query; - auto resultTime = BenchmarkShuffleElimination(config, session, query); + auto resultTime = BenchmarkShuffleElimination(config, session, resultType, query); Cout << "================================= FINALIZE ===============================\n"; auto deletionQuery = graph.GetSchema().MakeDropQuery(); @@ -342,41 +363,61 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { (*ctx.OS) << "\n"; } + std::string resultType = "SE"; + if (args.HasArg("result")) { + resultType = args.GetString("result"); + } + ui64 mcmcSteps = args.GetArgOrDefault("mcmcSteps", "100").GetValue(); for (ui64 n : args.GetArg("N")) { for (double alpha : args.GetArgOrDefault("alpha", "0.5")) { for (double theta : args.GetArgOrDefault("theta", "1.0")) { for (double logStdDev : args.GetArgOrDefault("logStdDev", "0.5")) { for (double logMean : args.GetArgOrDefault("logMean", "1.0")) { + Cout << "\n\n\n"; + Cout << "================================= METRICS ================================\n"; auto initialDegrees = GenerateLogNormalDegrees(n, logMean, logStdDev); + Cout << "initial degrees: " << joinVector(initialDegrees) << "\n"; + auto fixedDegrees = MakeGraphicConnected(initialDegrees); + Cout << "fixed degrees: " << joinVector(initialDegrees) << "\n"; + auto initialGraph = ConstructGraphHavelHakimi(fixedDegrees); + TRunningStatistics cummulative; + ui64 repeats = args.GetArgOrDefault("repeats", "1").GetValue(); for (ui64 i = 0; i < repeats; ++ i) { + Cout << "\n\n"; Cout << "Reproduce: 'N=" << n << "; alpha=" << alpha << "; theta=" << theta << "; logStdDev=" << logStdDev << "; logMean=" << logMean << "; seed=" << ctx.RNG.Serialize() << "'\n"; - ui64 seed = ctx.RNG.Serialize(); + // ui64 seed = ctx.RNG.Serialize(); TRelationGraph graph = initialGraph; MCMCRandomize(ctx.RNG, graph, mcmcSteps); graph.SetupKeysPitmanYor(ctx.RNG, TPitmanYorConfig{.Alpha = alpha, .Theta = theta}); - auto result = BenchmarkShuffleEliminationOnTopology(config, ctx.Session, graph); - if (!result) { - goto stop; - } - + try { + auto result = BenchmarkShuffleEliminationOnTopology(config, ctx.Session, resultType, graph); + if (!result) { + goto stop; + } - if (ctx.OS) { - (*ctx.OS) << n << "," << alpha << "," << theta << "," << logStdDev - << "," << logMean << "," << repeats << "," << seed << ","; - DumpBoxPlotToCSV(*ctx.OS, result->GetStatistics()); - (*ctx.OS) << "\n"; + cummulative.AddValues(*result); + } catch (...) { + Cout << "skipped\n"; + continue; } } + + if (ctx.OS) { + (*ctx.OS) << n << "," << alpha << "," << theta << "," << logStdDev + << "," << logMean << "," << repeats << "," << 0 << ","; + DumpBoxPlotToCSV(*ctx.OS, cummulative.GetStatistics()); + (*ctx.OS) << "\n"; + } } } } @@ -393,6 +434,11 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { (*ctx.OS) << "\n"; } + std::string resultType = "SE"; + if (args.HasArg("result")) { + resultType = args.GetString("result"); + } + for (ui64 n : args.GetArg("N")) { for (double alpha : args.GetArgOrDefault("alpha", "0.5")) { for (double theta : args.GetArgOrDefault("theta", "1.0")) { @@ -404,7 +450,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { TRelationGraph graph = TrivialTopology(ctx.RNG, n); graph.SetupKeysPitmanYor(ctx.RNG, TPitmanYorConfig{.Alpha = alpha, .Theta = theta}); - auto result = BenchmarkShuffleEliminationOnTopology(config, ctx.Session, graph); + auto result = BenchmarkShuffleEliminationOnTopology(config, ctx.Session, resultType, graph); if (!result) { goto stop; } From af3663154a9c86235499bc87577b7d192a4425ef Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Fri, 31 Oct 2025 05:18:11 +0000 Subject: [PATCH 26/43] Iterate number of nodes last to prevent early timeout --- ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp index e1070ae6c4c7..77d556957ea8 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp @@ -369,11 +369,11 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { } ui64 mcmcSteps = args.GetArgOrDefault("mcmcSteps", "100").GetValue(); - for (ui64 n : args.GetArg("N")) { - for (double alpha : args.GetArgOrDefault("alpha", "0.5")) { - for (double theta : args.GetArgOrDefault("theta", "1.0")) { - for (double logStdDev : args.GetArgOrDefault("logStdDev", "0.5")) { - for (double logMean : args.GetArgOrDefault("logMean", "1.0")) { + for (double alpha : args.GetArgOrDefault("alpha", "0.5")) { + for (double theta : args.GetArgOrDefault("theta", "1.0")) { + for (double logStdDev : args.GetArgOrDefault("logStdDev", "0.5")) { + for (double logMean : args.GetArgOrDefault("logMean", "1.0")) { + for (ui64 n : args.GetArg("N")) { Cout << "\n\n\n"; Cout << "================================= METRICS ================================\n"; auto initialDegrees = GenerateLogNormalDegrees(n, logMean, logStdDev); @@ -419,11 +419,11 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { (*ctx.OS) << "\n"; } } + stop:; } } } } - stop:; } template From c58bab4c68c66b9ab6f67a4ad143a10cf6286254 Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Fri, 31 Oct 2025 07:06:43 +0000 Subject: [PATCH 27/43] Collect a lot more data for plotting during benchmarks --- ydb/core/kqp/ut/common/kqp_benches.h | 8 +- ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp | 166 ++++++++++++------ 2 files changed, 114 insertions(+), 60 deletions(-) diff --git a/ydb/core/kqp/ut/common/kqp_benches.h b/ydb/core/kqp/ut/common/kqp_benches.h index 0776e05dbfcd..337f6f24d951 100644 --- a/ydb/core/kqp/ut/common/kqp_benches.h +++ b/ydb/core/kqp/ut/common/kqp_benches.h @@ -414,13 +414,13 @@ static std::string joinVector(const std::vector &data) { } -void DumpBoxPlotCSVHeader(IOutputStream &OS) { - OS << "median,Q1,Q3,IQR,MAD,min,max"; +std::string BoxPlotCSVHeader() { + return "median,Q1,Q3,IQR,MAD,min,max"; } template -void DumpBoxPlotToCSV(IOutputStream &OS, TStatistics stat) { - OS << stat.Median << "," << stat.Q1 << "," << stat.Q3 << "," << stat.IQR << "," << stat.MAD << "," << stat.Min << "," << stat.Max; +void DumpBoxPlotToCSV(IOutputStream &os, TStatistics stat) { + os << stat.Median << "," << stat.Q1 << "," << stat.Q3 << "," << stat.IQR << "," << stat.MAD << "," << stat.Min << "," << stat.Max; } diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp index 77d556957ea8..b7a4843bd5fb 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp @@ -1,3 +1,6 @@ +#include +#include +#include #include #include #include @@ -34,7 +37,9 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { } explainRes.GetIssues().PrintTo(Cout); - UNIT_ASSERT_VALUES_EQUAL(explainRes.GetStatus(), EStatus::SUCCESS); + if (explainRes.GetStatus() != EStatus::SUCCESS) { + throw std::runtime_error("Couldn't execute query!"); + } return *explainRes.GetStats()->GetPlan(); } @@ -98,19 +103,21 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { return stats; } - std::optional> BenchmarkShuffleElimination(TBenchmarkConfig config, NYdb::NQuery::TSession session, std::string resultType, const TString& query) { - Cout << resultType << "\n"; + std::optional>> BenchmarkShuffleElimination(TBenchmarkConfig config, NYdb::NQuery::TSession session, std::string resultType, const TString& query) { + std::map> results; std::optional> withoutCBO; if (resultType.contains("0")) { Cout << "--------------------------------- W/O CBO --------------------------------\n"; withoutCBO = BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/false, /*optLevel=*/0)); + results["0"] = withoutCBO->Cast(); + if (!withoutCBO) { return std::nullopt; } if (resultType == "0") { - return withoutCBO->Cast(); + return results; } } @@ -118,43 +125,44 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { if (resultType.contains("CBO")) { Cout << "--------------------------------- CBO-SE ---------------------------------\n"; withoutShuffleElimination = BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/false, /*optLevel=*/2)); - if (!withoutShuffleElimination) { - return std::nullopt; - } - if (resultType == "CBO") { - return withoutShuffleElimination->Cast(); + results["CBO"] = withoutShuffleElimination->Cast(); + if (resultType.contains("0")) { + results["CBO-0"] = (*withoutShuffleElimination - *withoutCBO).Cast(); } - if (resultType == "CBO-0") { - return (*withoutShuffleElimination - *withoutCBO).Cast(); + if (!withoutShuffleElimination) { + return std::nullopt; } + + return results; } std::optional> withShuffleElimination; if (resultType.contains("SE")) { Cout << "--------------------------------- CBO+SE ---------------------------------\n"; withShuffleElimination = BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/true, /*optLevel=*/2)); - if (!withShuffleElimination) { - return std::nullopt; - } - if (resultType == "SE") { - return withShuffleElimination->Cast(); + results["SE"] = withShuffleElimination->Cast(); + if (resultType.contains("0")) { + results["SE-0"] = (*withShuffleElimination - *withoutCBO).Cast(); } - if (resultType == "SE-0") { - return (*withShuffleElimination - *withoutCBO).Cast(); + if (!withShuffleElimination) { + return std::nullopt; } + + return results; } Cout << "--------------------------------------------------------------------------\n"; - if (resultType == "SE-0/CBO-0") { - return (*withShuffleElimination - *withoutCBO) / (*withoutShuffleElimination - *withoutCBO); - } else { - return *withShuffleElimination / *withoutShuffleElimination; + results["SE"] = *withShuffleElimination / *withoutShuffleElimination; + if (resultType.contains("0")) { + results["SE-0"] = (*withShuffleElimination - *withoutCBO) / (*withoutShuffleElimination - *withoutCBO); } + + return results; } std::unique_ptr GetCBOTestsYDB(TString stats, TDuration compilationTimeout) { @@ -179,7 +187,8 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { return std::make_unique(serverSettings); } - std::optional> BenchmarkShuffleEliminationOnTopology(TBenchmarkConfig config, NYdb::NQuery::TSession session, std::string resultType, TRelationGraph graph) { + std::optional>> + BenchmarkShuffleEliminationOnTopology(TBenchmarkConfig config, NYdb::NQuery::TSession session, std::string resultType, TRelationGraph graph) { Cout << "================================= CREATE =================================\n"; graph.DumpGraph(Cout); @@ -330,10 +339,12 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { NYdb::NQuery::TSession Session; TRNG RNG; - std::optional OS; + std::string OutputDir; + std::map Streams = {}; + }; - TTestContext CreateTestContext(TArgs args, uint64_t state = 0, std::string outputFile = "") { + TTestContext CreateTestContext(TArgs args, uint64_t state = 0, std::string outputDir = "") { TRNG rng = TRNG::Deserialize(state); auto numTablesRanged = args.GetArg("N"); @@ -348,26 +359,61 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { auto db = kikimr->GetQueryClient(); auto session = db.GetSession().GetValueSync().GetSession(); - std::optional os = std::nullopt; - if (!outputFile.empty()) { - os = TUnbufferedFileOutput(outputFile); + return { std::move(kikimr), std::move(db), std::move(session), std::move(rng), outputDir }; + } + + void WriteAllStats(TTestContext &ctx, const std::string& prefix, + const std::string& header, const std::string& params, + const std::map>& stats) { + + if (ctx.OutputDir.empty()) { + return; } - return { std::move(kikimr), std::move(db), std::move(session), std::move(rng), std::move(os) }; + if (!std::filesystem::exists(ctx.OutputDir)) { + std::filesystem::create_directory(ctx.OutputDir); + } + + for (const auto &[key, stats]: stats) { + std::string name = prefix + key; + + if (!ctx.Streams.contains(name)) { + auto filename = TString(ctx.OutputDir + "/" + name + ".csv"); + auto &OS = ctx.Streams.emplace(name, TUnbufferedFileOutput(filename)).first->second; + OS << header; + } + + auto &OS = ctx.Streams.find(name)->second; + + OS << params; + DumpBoxPlotToCSV(OS, stats.GetStatistics()); + OS << "\n"; + } } - void RunMCMC(TTestContext &ctx, TBenchmarkConfig config, TArgs args) { - if (ctx.OS) { - (*ctx.OS) << "N,alpha,theta,logStdDev,logMean,repeats,seed,"; - DumpBoxPlotCSVHeader(*ctx.OS); - (*ctx.OS) << "\n"; + void AccumulateAllStats(std::map>& cummulative, + const std::map>& stats) { + for (auto &[key, stat] : stats) { + if (!cummulative.contains(key)) { + cummulative.emplace(key, stat); + } + + cummulative.find(key)->second.AddValues(stat); } + } + + void RunMCMC(TTestContext &ctx, TBenchmarkConfig config, TArgs args) { + std::string header = "N,i,repeat,alpha,theta,logStdDev,logMean,repeats,seed," + BoxPlotCSVHeader() + "\n"; + std::string headerCummulative = "N,i,alpha,theta,logStdDev,logMean,repeats" + BoxPlotCSVHeader() + "\n"; std::string resultType = "SE"; if (args.HasArg("result")) { resultType = args.GetString("result"); } + ui32 globalNum = 0; + ui32 cummulativeNum = 0; + ui64 mcmcSteps = args.GetArgOrDefault("mcmcSteps", "100").GetValue(); for (double alpha : args.GetArgOrDefault("alpha", "0.5")) { for (double theta : args.GetArgOrDefault("theta", "1.0")) { @@ -384,7 +430,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { auto initialGraph = ConstructGraphHavelHakimi(fixedDegrees); - TRunningStatistics cummulative; + std::map> cummulative; ui64 repeats = args.GetArgOrDefault("repeats", "1").GetValue(); for (ui64 i = 0; i < repeats; ++ i) { @@ -393,7 +439,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { << theta << "; logStdDev=" << logStdDev << "; logMean=" << logMean << "; seed=" << ctx.RNG.Serialize() << "'\n"; - // ui64 seed = ctx.RNG.Serialize(); + ui64 seed = ctx.RNG.Serialize(); TRelationGraph graph = initialGraph; MCMCRandomize(ctx.RNG, graph, mcmcSteps); @@ -405,19 +451,27 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { goto stop; } - cummulative.AddValues(*result); - } catch (...) { - Cout << "skipped\n"; + AccumulateAllStats(cummulative, *result); + + std::stringstream params; + params << n << "," << (globalNum ++) << "," << i + << "," << alpha << "," << theta << "," << logStdDev + << "," << logMean << "," << repeats << "," << seed << ","; + + WriteAllStats(ctx, "", header, params.str(), *result); + } catch (std::exception &exc) { + Cout << "Skipped run: " << exc.what() << "\n"; continue; } } - if (ctx.OS) { - (*ctx.OS) << n << "," << alpha << "," << theta << "," << logStdDev - << "," << logMean << "," << repeats << "," << 0 << ","; - DumpBoxPlotToCSV(*ctx.OS, cummulative.GetStatistics()); - (*ctx.OS) << "\n"; - } + std::stringstream params; + params << n << "," << (cummulativeNum ++) + << "," << alpha << "," << theta + << "," << logStdDev << "," << logMean + << "," << repeats << ","; + + WriteAllStats(ctx, "cummulative-", headerCummulative, params.str(), cummulative); } stop:; } @@ -428,11 +482,11 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { template void RunTrivialTopology(TTestContext &ctx, TBenchmarkConfig config, TArgs args) { - if (ctx.OS) { - (*ctx.OS) << "N,alpha,theta,seed,"; - DumpBoxPlotCSVHeader(*ctx.OS); - (*ctx.OS) << "\n"; - } + // if (ctx.OS) { + // (*ctx.OS) << "N,alpha,theta,seed,"; + // DumpBoxPlotCSVHeader(*ctx.OS); + // (*ctx.OS) << "\n"; + // } std::string resultType = "SE"; if (args.HasArg("result")) { @@ -455,11 +509,11 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { goto stop; } - if (ctx.OS) { - (*ctx.OS) << n << "," << alpha << "," << theta << "," << seed << ","; - DumpBoxPlotToCSV(*ctx.OS, result->GetStatistics()); - (*ctx.OS) << "\n"; - } + // if (ctx.OS) { + // (*ctx.OS) << n << "," << alpha << "," << theta << "," << seed << ","; + // DumpBoxPlotToCSV(*ctx.OS, result->GetStatistics()); + // (*ctx.OS) << "\n"; + // } } } } @@ -485,7 +539,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { state = args.GetArg("seed").GetValue(); } - TTestContext ctx = CreateTestContext(args, state, GetTestParam("SAVE_FILE")); + TTestContext ctx = CreateTestContext(args, state, GetTestParam("SAVE_DIR")); if (topology == "mcmc") { RunMCMC(ctx, config, args); From 51adb62f286e373a62fb620956ef074e9940f219 Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Fri, 31 Oct 2025 07:37:22 +0000 Subject: [PATCH 28/43] Reuse more comprehensive info collection for simpler topologies --- .../ut/join/kqp_join_topology_generator.cpp | 8 +- ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp | 91 +++++++++++++------ 2 files changed, 65 insertions(+), 34 deletions(-) diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp index 485f73245623..e7b3f4f0752c 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp @@ -473,8 +473,8 @@ std::vector SampleFromPMF(const std::vector& probabilities, return degrees; } -std::vector GenerateLogNormalDegrees(int numVertices, double logMean, - double logStdDev, int minDegree, +std::vector GenerateLogNormalDegrees(int numVertices, double mu, + double sigma, int minDegree, int maxDegree) { if (maxDegree == -1) maxDegree = numVertices - 1; @@ -487,10 +487,10 @@ std::vector GenerateLogNormalDegrees(int numVertices, double logMean, double x = (double)k; double logX = std::log(x); - double z = (logX - logMean) / logStdDev; + double z = (logX - mu) / sigma; // PDF: (1/(x*σ*√(2π))) * exp(-(ln(x)-μ)²/(2σ²)) - probabilities[k - minDegree] = (1.0 / (x * logStdDev * std::sqrt(2.0 * M_PI))) * + probabilities[k - minDegree] = (1.0 / (x * sigma * std::sqrt(2.0 * M_PI))) * std::exp(-0.5 * z * z); } diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp index b7a4843bd5fb..7276c3fb28dd 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp @@ -135,7 +135,9 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { return std::nullopt; } - return results; + if (resultType == "CBO" || resultType == "CBO-0") { + return results; + } } std::optional> withShuffleElimination; @@ -152,7 +154,9 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { return std::nullopt; } - return results; + if (resultType == "SE" || resultType == "SE-0") { + return results; + } } Cout << "--------------------------------------------------------------------------\n"; @@ -206,17 +210,23 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { Cout << "================================= BENCHMARK ==============================\n"; TString query = graph.MakeQuery(); Cout << query; - auto resultTime = BenchmarkShuffleElimination(config, session, resultType, query); Cout << "================================= FINALIZE ===============================\n"; auto deletionQuery = graph.GetSchema().MakeDropQuery(); Cout << deletionQuery; - if (!ExecuteQuery(session, deletionQuery)) { // TODO: this is really bad probably? - return std::nullopt; - } - Cout << "==========================================================================\n"; - return resultTime; + try { + auto resultTime = BenchmarkShuffleElimination(config, session, resultType, query); + ExecuteQuery(session, deletionQuery); + Cout << "==========================================================================\n"; + + return resultTime; + } catch (std::exception &exc) { + ExecuteQuery(session, deletionQuery); + Cout << "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! FAILED !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"; + + throw std::runtime_error(std::string("Benchmark failed with '") + exc.what() + "'"); + } } @@ -402,9 +412,10 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { } } - void RunMCMC(TTestContext &ctx, TBenchmarkConfig config, TArgs args) { - std::string header = "N,i,repeat,alpha,theta,logStdDev,logMean,repeats,seed," + BoxPlotCSVHeader() + "\n"; - std::string headerCummulative = "N,i,alpha,theta,logStdDev,logMean,repeats" + BoxPlotCSVHeader() + "\n"; + template + void RunBenches(TTestContext &ctx, TBenchmarkConfig config, TArgs args, TGenerateTopology generateTopology) { + std::string header = "N,i,repeat,alpha,theta,sigma,mu,repeats,seed," + BoxPlotCSVHeader() + "\n"; + std::string headerCummulative = "N,i,alpha,theta,sigma,mu,repeats," + BoxPlotCSVHeader() + "\n"; std::string resultType = "SE"; if (args.HasArg("result")) { @@ -417,18 +428,11 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { ui64 mcmcSteps = args.GetArgOrDefault("mcmcSteps", "100").GetValue(); for (double alpha : args.GetArgOrDefault("alpha", "0.5")) { for (double theta : args.GetArgOrDefault("theta", "1.0")) { - for (double logStdDev : args.GetArgOrDefault("logStdDev", "0.5")) { - for (double logMean : args.GetArgOrDefault("logMean", "1.0")) { + for (double sigma : args.GetArgOrDefault("sigma", "0.5")) { + for (double mu : args.GetArgOrDefault("mu", "1.0")) { for (ui64 n : args.GetArg("N")) { Cout << "\n\n\n"; - Cout << "================================= METRICS ================================\n"; - auto initialDegrees = GenerateLogNormalDegrees(n, logMean, logStdDev); - Cout << "initial degrees: " << joinVector(initialDegrees) << "\n"; - - auto fixedDegrees = MakeGraphicConnected(initialDegrees); - Cout << "fixed degrees: " << joinVector(initialDegrees) << "\n"; - - auto initialGraph = ConstructGraphHavelHakimi(fixedDegrees); + auto initialGraph = generateTopology(ctx.RNG, n, mu, sigma); std::map> cummulative; @@ -436,13 +440,16 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { for (ui64 i = 0; i < repeats; ++ i) { Cout << "\n\n"; Cout << "Reproduce: 'N=" << n << "; alpha=" << alpha << "; theta=" - << theta << "; logStdDev=" << logStdDev << "; logMean=" << logMean + << theta << "; sigma=" << sigma << "; mu=" << mu << "; seed=" << ctx.RNG.Serialize() << "'\n"; ui64 seed = ctx.RNG.Serialize(); TRelationGraph graph = initialGraph; - MCMCRandomize(ctx.RNG, graph, mcmcSteps); + if (i != 0) { + MCMCRandomize(ctx.RNG, graph, mcmcSteps); + } + graph.SetupKeysPitmanYor(ctx.RNG, TPitmanYorConfig{.Alpha = alpha, .Theta = theta}); try { @@ -455,8 +462,8 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { std::stringstream params; params << n << "," << (globalNum ++) << "," << i - << "," << alpha << "," << theta << "," << logStdDev - << "," << logMean << "," << repeats << "," << seed << ","; + << "," << alpha << "," << theta << "," << sigma + << "," << mu << "," << repeats << "," << seed << ","; WriteAllStats(ctx, "", header, params.str(), *result); } catch (std::exception &exc) { @@ -468,7 +475,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { std::stringstream params; params << n << "," << (cummulativeNum ++) << "," << alpha << "," << theta - << "," << logStdDev << "," << logMean + << "," << sigma << "," << mu << "," << repeats << ","; WriteAllStats(ctx, "cummulative-", headerCummulative, params.str(), cummulative); @@ -541,14 +548,38 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { TTestContext ctx = CreateTestContext(args, state, GetTestParam("SAVE_DIR")); + auto mcmc = [&]([[maybe_unused]] TRNG rng, ui32 n, double mu, double sigma) { + Cout << "================================= METRICS ================================\n"; + auto initialDegrees = GenerateLogNormalDegrees(n, mu, sigma); + Cout << "initial degrees: " << joinVector(initialDegrees) << "\n"; + + auto fixedDegrees = MakeGraphicConnected(initialDegrees); + Cout << "fixed degrees: " << joinVector(initialDegrees) << "\n"; + + auto initialGraph = ConstructGraphHavelHakimi(fixedDegrees); + return initialGraph; + }; + + auto star = [&]([[maybe_unused]] TRNG rng, ui32 n, [[maybe_unused]] double mu, [[maybe_unused]] double sigma) { + return GenerateStar(rng, n); + }; + + auto path = [&]([[maybe_unused]] TRNG rng, ui32 n, [[maybe_unused]] double mu, [[maybe_unused]] double sigma) { + return GenerateLine(rng, n); + }; + + auto clique = [&]([[maybe_unused]] TRNG rng, ui32 n, [[maybe_unused]] double mu, [[maybe_unused]] double sigma) { + return GenerateLine(rng, n); + }; + if (topology == "mcmc") { - RunMCMC(ctx, config, args); + RunBenches(ctx, config, args, mcmc); } else if (topology == "star") { - RunTrivialTopology(ctx, config, args); + RunBenches(ctx, config, args, star); } else if (topology == "path") { - RunTrivialTopology(ctx, config, args); + RunBenches(ctx, config, args, path); } else if (topology == "clique") { - RunTrivialTopology(ctx, config, args); + RunBenches(ctx, config, args, clique); } } From ea0fdbfc2e6c2210b1ac58973e6e2085eebd2d44 Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Fri, 31 Oct 2025 08:20:29 +0000 Subject: [PATCH 29/43] Implement repeated regeneration with same params + fix bug with serialization --- ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp | 69 ++++++++++--------- 1 file changed, 36 insertions(+), 33 deletions(-) diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp index 7276c3fb28dd..85b0d3b51221 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp @@ -161,9 +161,9 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { Cout << "--------------------------------------------------------------------------\n"; - results["SE"] = *withShuffleElimination / *withoutShuffleElimination; + results["SE-div-CBO"] = *withShuffleElimination / *withoutShuffleElimination; if (resultType.contains("0")) { - results["SE-0"] = (*withShuffleElimination - *withoutCBO) / (*withoutShuffleElimination - *withoutCBO); + results["SE-0-div-CBO-0"] = (*withShuffleElimination - *withoutCBO) / (*withoutShuffleElimination - *withoutCBO); } return results; @@ -415,7 +415,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { template void RunBenches(TTestContext &ctx, TBenchmarkConfig config, TArgs args, TGenerateTopology generateTopology) { std::string header = "N,i,repeat,alpha,theta,sigma,mu,repeats,seed," + BoxPlotCSVHeader() + "\n"; - std::string headerCummulative = "N,i,alpha,theta,sigma,mu,repeats," + BoxPlotCSVHeader() + "\n"; + std::string headerCummulative = "N,i,alpha,theta,sigma,mu,gen," + BoxPlotCSVHeader() + "\n"; std::string resultType = "SE"; if (args.HasArg("result")) { @@ -431,44 +431,47 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { for (double sigma : args.GetArgOrDefault("sigma", "0.5")) { for (double mu : args.GetArgOrDefault("mu", "1.0")) { for (ui64 n : args.GetArg("N")) { - Cout << "\n\n\n"; - auto initialGraph = generateTopology(ctx.RNG, n, mu, sigma); - std::map> cummulative; - ui64 repeats = args.GetArgOrDefault("repeats", "1").GetValue(); - for (ui64 i = 0; i < repeats; ++ i) { - Cout << "\n\n"; - Cout << "Reproduce: 'N=" << n << "; alpha=" << alpha << "; theta=" - << theta << "; sigma=" << sigma << "; mu=" << mu - << "; seed=" << ctx.RNG.Serialize() << "'\n"; - - ui64 seed = ctx.RNG.Serialize(); + ui64 generationRepeats = args.GetArgOrDefault("gen", "1").GetValue(); + for (ui64 j = 0; j < generationRepeats; ++ j) { + Cout << "\n\n\n"; + auto initialGraph = generateTopology(ctx.RNG, n, mu, sigma); - TRelationGraph graph = initialGraph; - if (i != 0) { - MCMCRandomize(ctx.RNG, graph, mcmcSteps); - } + ui64 repeats = args.GetArgOrDefault("repeats", "1").GetValue(); + for (ui64 i = 0; i < repeats; ++ i) { + Cout << "\n\n"; + Cout << "Reproduce: 'N=" << n << "; alpha=" << alpha << "; theta=" + << theta << "; sigma=" << sigma << "; mu=" << mu + << "; seed=" << ctx.RNG.Serialize() << "'\n"; - graph.SetupKeysPitmanYor(ctx.RNG, TPitmanYorConfig{.Alpha = alpha, .Theta = theta}); + ui64 seed = ctx.RNG.Serialize(); - try { - auto result = BenchmarkShuffleEliminationOnTopology(config, ctx.Session, resultType, graph); - if (!result) { - goto stop; + TRelationGraph graph = initialGraph; + if (i != 0) { + MCMCRandomize(ctx.RNG, graph, mcmcSteps); } - AccumulateAllStats(cummulative, *result); + graph.SetupKeysPitmanYor(ctx.RNG, TPitmanYorConfig{.Alpha = alpha, .Theta = theta}); - std::stringstream params; - params << n << "," << (globalNum ++) << "," << i - << "," << alpha << "," << theta << "," << sigma - << "," << mu << "," << repeats << "," << seed << ","; + try { + auto result = BenchmarkShuffleEliminationOnTopology(config, ctx.Session, resultType, graph); + if (!result) { + goto stop; + } - WriteAllStats(ctx, "", header, params.str(), *result); - } catch (std::exception &exc) { - Cout << "Skipped run: " << exc.what() << "\n"; - continue; + AccumulateAllStats(cummulative, *result); + + std::stringstream params; + params << n << "," << (globalNum ++) << "," << i + << "," << alpha << "," << theta << "," << sigma + << "," << mu << "," << repeats << "," << seed << ","; + + WriteAllStats(ctx, "", header, params.str(), *result); + } catch (std::exception &exc) { + Cout << "Skipped run: " << exc.what() << "\n"; + continue; + } } } @@ -476,7 +479,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { params << n << "," << (cummulativeNum ++) << "," << alpha << "," << theta << "," << sigma << "," << mu - << "," << repeats << ","; + << "," << generationRepeats << ","; WriteAllStats(ctx, "cummulative-", headerCummulative, params.str(), cummulative); } From df26fdf2bf7c0a64be46e1413654358bd74fe8e1 Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Fri, 31 Oct 2025 11:45:20 +0000 Subject: [PATCH 30/43] Fix cbo unittests after MakeNativeOptimizerNew args were updated --- ydb/library/yql/dq/opt/ut/dq_cbo_ut.cpp | 13 +++++++------ ydb/library/yql/dq/opt/ut/dq_opt_hypergraph_ut.cpp | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/ydb/library/yql/dq/opt/ut/dq_cbo_ut.cpp b/ydb/library/yql/dq/opt/ut/dq_cbo_ut.cpp index 3c8b700e5228..f50eeb9120fd 100644 --- a/ydb/library/yql/dq/opt/ut/dq_cbo_ut.cpp +++ b/ydb/library/yql/dq/opt/ut/dq_cbo_ut.cpp @@ -35,7 +35,7 @@ Y_UNIT_TEST(Empty) { TBaseProviderContext pctx; TExprContext dummyCtx; - TCBOSettings settings{}; + TOptimizerSettings settings{}; std::unique_ptr optimizer = std::unique_ptr(MakeNativeOptimizerNew(pctx, settings, dummyCtx, false)); } @@ -43,7 +43,7 @@ Y_UNIT_TEST(JoinSearch2Rels) { TBaseProviderContext pctx; TExprContext dummyCtx; - TCBOSettings settings{}; + TOptimizerSettings settings{}; std::unique_ptr optimizer = std::unique_ptr(MakeNativeOptimizerNew(pctx, settings, dummyCtx, false)); auto rel1 = std::make_shared( @@ -79,7 +79,7 @@ Y_UNIT_TEST(JoinSearch3Rels) { TBaseProviderContext pctx; TExprContext dummyCtx; - TCBOSettings settings{}; + TOptimizerSettings settings{}; std::unique_ptr optimizer = std::unique_ptr(MakeNativeOptimizerNew(pctx, settings, dummyCtx, false)); auto rel1 = std::make_shared("a", @@ -128,7 +128,7 @@ Y_UNIT_TEST(JoinSearchYQL19363) { TBaseProviderContext pctx; TExprContext dummyCtx; - TCBOSettings settings{}; + TOptimizerSettings settings{}; std::unique_ptr optimizer = std::unique_ptr(MakeNativeOptimizerNew(pctx, settings, dummyCtx, false)); TString relName1 = "a,b.c"; @@ -237,7 +237,7 @@ Y_UNIT_TEST(JoinSearchYT24403) { TMockProviderContextYT24403 pctx; TExprContext dummyCtx; - TCBOSettings settings{}; + TOptimizerSettings settings{}; std::unique_ptr optimizer = std::unique_ptr(MakeNativeOptimizerNew(pctx, settings, dummyCtx, false)); const TString relName1 = "a"; @@ -378,7 +378,8 @@ Y_UNIT_TEST(DqOptimizeEquiJoinWithCostsNative) { TExprContext ctx; TBaseProviderContext pctx; std::function optFactory = [&]() { - return MakeNativeOptimizerNew(pctx, TOptimizerSettings{.MaxDPHypDPTableSize = 100000}, ctx, false); + TOptimizerSettings settings{}; + return MakeNativeOptimizerNew(pctx, settings, ctx, false); }; _DqOptimizeEquiJoinWithCosts(optFactory, ctx); } diff --git a/ydb/library/yql/dq/opt/ut/dq_opt_hypergraph_ut.cpp b/ydb/library/yql/dq/opt/ut/dq_opt_hypergraph_ut.cpp index 1c1795fd0fb6..f3ffbc9b92ff 100644 --- a/ydb/library/yql/dq/opt/ut/dq_opt_hypergraph_ut.cpp +++ b/ydb/library/yql/dq/opt/ut/dq_opt_hypergraph_ut.cpp @@ -53,7 +53,7 @@ std::shared_ptr Enumerate(const std::shared_ptr(MakeNativeOptimizerNew(ctx, settings, dummyCtx, false)); From d6d7ec4b2b70e2f2a570ef5ebacd410f0225cee0 Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Sat, 1 Nov 2025 13:17:49 +0000 Subject: [PATCH 31/43] Large refactoring of topology benchmark --- ydb/core/kqp/ut/common/kqp_arg_parser.h | 8 + ydb/core/kqp/ut/common/kqp_benches.h | 16 +- .../ut/join/kqp_join_topology_generator.cpp | 47 +-- .../kqp/ut/join/kqp_join_topology_generator.h | 53 +++- ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp | 279 ++++++++---------- 5 files changed, 208 insertions(+), 195 deletions(-) diff --git a/ydb/core/kqp/ut/common/kqp_arg_parser.h b/ydb/core/kqp/ut/common/kqp_arg_parser.h index 9fe4be03630b..febc536e9467 100644 --- a/ydb/core/kqp/ut/common/kqp_arg_parser.h +++ b/ydb/core/kqp/ut/common/kqp_arg_parser.h @@ -119,6 +119,14 @@ class TArgs { return Values_[key]; } + std::string GetStringOrDefault(std::string key, std::string defaultValue) { + if (HasArg(key)) { + return GetString(key); + } + + return defaultValue; + } + template auto GetArg(std::string key) { return ParseRangedValue(GetString(key)); diff --git a/ydb/core/kqp/ut/common/kqp_benches.h b/ydb/core/kqp/ut/common/kqp_benches.h index 337f6f24d951..d9ac9fcd5930 100644 --- a/ydb/core/kqp/ut/common/kqp_benches.h +++ b/ydb/core/kqp/ut/common/kqp_benches.h @@ -333,6 +333,18 @@ class TRunningStatistics { } } + template + TRunningStatistics Reject(Lambda &&shouldReject) const { + TRunningStatistics stats; + IterateElements([&](TValue value) { + if (!shouldReject(value)) { + stats.AddValue(value); + } + }); + + return stats; + } + TRunningStatistics operator-(TRunningStatistics rhsStats) const { TRunningStatistics stats; IterateElements([&](TValue lhs) { @@ -398,7 +410,7 @@ class TRunningStatistics { }; template -static std::string joinVector(const std::vector &data) { +static std::string joinVector(const std::vector &data, std::string delimeter = ";") { if (data.empty()) { return ""; } @@ -407,7 +419,7 @@ static std::string joinVector(const std::vector &data) { str += std::to_string(data[0]); for (ui32 i = 1; i < data.size(); ++ i) { - str += ";" + std::to_string(data[i]); + str += delimeter + std::to_string(data[i]); } return str; diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp index e7b3f4f0752c..a737e71fc7bd 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp @@ -10,46 +10,6 @@ namespace NKikimr::NKqp { -class TLexicographicalNameGenerator { -public: - static std::string getName(unsigned ID, bool lowerCase = true) { - if (ID < Base_) - return std::string(1, fromDigit(ID, lowerCase)); - - ID -= Base_; - - unsigned count = 1; - unsigned step = Base_; - for (; ID >= step;) { - ID -= step; - step *= step; - count *= 2; - } - - std::string result(count, fromDigit(Base_ - 1, lowerCase)); - return result + fromNumber(ID, result.size(), lowerCase); - } - -private: - static std::string fromNumber(unsigned number, unsigned size, bool lowerCase) { - std::string stringified = ""; - for (unsigned i = 0; i < size; ++ i) { - stringified.push_back(fromDigit(number % Base_, lowerCase)); - number /= Base_; - } - - return std::string(stringified.rbegin(), stringified.rend()); - } - - static char fromDigit(unsigned value, bool lowerCase) { - assert(0 <= value && value < Base_); - return (lowerCase ? 'a' : 'A') + value; - } - - static constexpr unsigned Base_ = 'z' - 'a' + 1; -}; - - static std::string getTableName(unsigned tableID) { return TLexicographicalNameGenerator::getName(tableID, /*lowerCase=*/false); } @@ -659,10 +619,13 @@ TRelationGraph ConstructGraphHavelHakimi(std::vector degrees) { return graph; } -void MCMCRandomize(TRNG &mt, TRelationGraph& graph, int numSwaps) { +void MCMCRandomize(TRNG &mt, TRelationGraph& graph) { std::uniform_int_distribution<> nodeDist(0, graph.GetN() - 1); std::uniform_real_distribution<> probDist(0.0, 1.0); + ui32 numEdges = graph.GetEdges(); + ui32 numSwaps = numEdges * log(numEdges); + auto &adjacency = graph.GetAdjacencyList(); const double TEMP_START = 5.0; @@ -670,7 +633,7 @@ void MCMCRandomize(TRNG &mt, TRelationGraph& graph, int numSwaps) { const double CONNECTIVITY_PENALTY = 20.0; const int MAX_ATTEMPTS = numSwaps * 10; - int successfulSwaps = 0; + ui32 successfulSwaps = 0; int attempt = 0; while (attempt < MAX_ATTEMPTS && (successfulSwaps < numSwaps || !graph.IsConnected())) { diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_generator.h b/ydb/core/kqp/ut/join/kqp_join_topology_generator.h index 8f25ce8dbf03..a19fe9d5f240 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_generator.h +++ b/ydb/core/kqp/ut/join/kqp_join_topology_generator.h @@ -16,6 +16,48 @@ namespace NKikimr::NKqp { using TRNG = TSerializableMT19937; +class TLexicographicalNameGenerator { +public: + static std::string getName(unsigned ID, bool lowerCase = true) { + if (ID < Base_) + return std::string(1, fromDigit(ID, lowerCase)); + + ID -= Base_; + + unsigned count = 1; + unsigned step = Base_; + for (; ID >= step;) { + ID -= step; + step *= step; + count *= 2; + } + + std::string result(count, fromDigit(Base_ - 1, lowerCase)); + return result + fromNumber(ID, result.size(), lowerCase); + } + +private: + static std::string fromNumber(unsigned number, unsigned size, bool lowerCase) { + std::string stringified = ""; + for (unsigned i = 0; i < size; ++ i) { + stringified.push_back(fromDigit(number % Base_, lowerCase)); + number /= Base_; + } + + return std::string(stringified.rbegin(), stringified.rend()); + } + + static char fromDigit(unsigned value, bool lowerCase) { + assert(0 <= value && value < Base_); + return (lowerCase ? 'a' : 'A') + value; + } + + static constexpr unsigned Base_ = 'z' - 'a' + 1; +}; + + + + struct TPitmanYorConfig { double Alpha; double Theta; @@ -177,6 +219,15 @@ class TRelationGraph { void SetupKeysPitmanYor(TRNG &mt, TPitmanYorConfig config); + ui32 GetEdges() { + ui32 numEdges = 0; + for (auto edges : AdjacencyList_) { + numEdges += edges.size(); + } + + return numEdges; + } + public: struct TEdge { unsigned Target; @@ -233,7 +284,7 @@ TRelationGraph ConstructGraphHavelHakimi(std::vector degrees); std::vector MakeGraphicConnected(std::vector degrees); -void MCMCRandomize(TRNG &mt, TRelationGraph& graph, int numSwaps); +void MCMCRandomize(TRNG &mt, TRelationGraph& graph); TRelationGraph GenerateLine(TRNG &rng, unsigned numNodes); TRelationGraph GenerateStar(TRNG &rng, unsigned numNodes); diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp index 85b0d3b51221..5b82f871b594 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -163,7 +164,17 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { results["SE-div-CBO"] = *withShuffleElimination / *withoutShuffleElimination; if (resultType.contains("0")) { - results["SE-0-div-CBO-0"] = (*withShuffleElimination - *withoutCBO) / (*withoutShuffleElimination - *withoutCBO); + if (withShuffleElimination->GetMedian() > withoutShuffleElimination->GetMedian() || + withShuffleElimination->GetMedian() > withoutCBO->GetMedian()) { + + auto adjustedWithTime = (*withShuffleElimination - *withoutCBO).Reject([](i64 value) { return value < 0; }); + auto adjustedWithoutTime = (*withoutShuffleElimination - *withoutCBO).Reject([](i64 value) { return value < 0; }); + + auto adjustedRatio = adjustedWithTime / adjustedWithoutTime; + adjustedRatio = adjustedRatio.Reject([](double value) { return value < 1.0; }); + + results["SE-0-div-CBO-0"] = adjustedRatio; + } } return results; @@ -314,35 +325,6 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { }; } - class TMCMCTester : public TTopologyTester { - public: - void Initialize(TArgs args) override { - N_ = args.GetArg("N").GetValue(); - - auto initialDegrees = GenerateLogNormalDegrees(N_); - auto fixedDegrees = MakeGraphicConnected(initialDegrees); - InitialGraph_ = ConstructGraphHavelHakimi(fixedDegrees); - } - - TRelationGraph ProduceGraph(TRNG &rng) override { - TRelationGraph graph = InitialGraph_; - MCMCRandomize(rng, graph, /*MCMCSteps*/100); - return graph; - } - - void DumpParamsHeader(IOutputStream &OS) override { - OS << "N,"; - } - - void DumpParams(IOutputStream &OS) override { - OS << N_ << ","; - } - - private: - TRelationGraph InitialGraph_; - ui32 N_; - }; - struct TTestContext { std::unique_ptr Runner; NYdb::NQuery::TQueryClient QueryClient; @@ -372,6 +354,25 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { return { std::move(kikimr), std::move(db), std::move(session), std::move(rng), outputDir }; } + std::string WriteGraph(TTestContext &ctx, ui32 graphID, const TRelationGraph &graph) { + if (ctx.OutputDir.empty()) { + return ""; + } + + std::string graphDir = ctx.OutputDir + "/graphs"; + if (!std::filesystem::exists(graphDir)) { + std::filesystem::create_directories(graphDir); + } + + std::string graphName = TLexicographicalNameGenerator::getName(graphID); + + TString filename = graphDir + "/" + graphName + ".dot"; + auto os = TUnbufferedFileOutput(filename); + graph.DumpGraph(os); + + return graphName; + } + void WriteAllStats(TTestContext &ctx, const std::string& prefix, const std::string& header, const std::string& params, const std::map>& stats) { @@ -381,7 +382,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { } if (!std::filesystem::exists(ctx.OutputDir)) { - std::filesystem::create_directory(ctx.OutputDir); + std::filesystem::create_directories(ctx.OutputDir); } for (const auto &[key, stats]: stats) { @@ -389,15 +390,15 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { if (!ctx.Streams.contains(name)) { auto filename = TString(ctx.OutputDir + "/" + name + ".csv"); - auto &OS = ctx.Streams.emplace(name, TUnbufferedFileOutput(filename)).first->second; - OS << header; + auto &os = ctx.Streams.emplace(name, TUnbufferedFileOutput(filename)).first->second; + os << header << "\n"; } - auto &OS = ctx.Streams.find(name)->second; + auto &os = ctx.Streams.find(name)->second; - OS << params; - DumpBoxPlotToCSV(OS, stats.GetStatistics()); - OS << "\n"; + os << params << ","; + DumpBoxPlotToCSV(os, stats.GetStatistics()); + os << "\n"; } } @@ -412,44 +413,107 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { } } - template - void RunBenches(TTestContext &ctx, TBenchmarkConfig config, TArgs args, TGenerateTopology generateTopology) { - std::string header = "N,i,repeat,alpha,theta,sigma,mu,repeats,seed," + BoxPlotCSVHeader() + "\n"; - std::string headerCummulative = "N,i,alpha,theta,sigma,mu,gen," + BoxPlotCSVHeader() + "\n"; - std::string resultType = "SE"; - if (args.HasArg("result")) { - resultType = args.GetString("result"); + using TTopologyFunction = TRelationGraph (*) (TRNG rng, ui32 n, double mu, double sigma); + template + TTopologyFunction GetTrivialTopology() { + return []([[maybe_unused]] TRNG rng, ui32 n, [[maybe_unused]] double mu, [[maybe_unused]] double sigma) { + return TTrivialTopologyGenerator(rng, n); + }; + } + + TTopologyFunction GetTopology(std::string topologyName) { + if (topologyName == "star") { + return GetTrivialTopology(); } - ui32 globalNum = 0; - ui32 cummulativeNum = 0; + if (topologyName == "path") { + return GetTrivialTopology(); + } + + if (topologyName == "clique") { + return GetTrivialTopology(); + } + + if (topologyName == "random-tree") { + return GetTrivialTopology(); + } + + if (topologyName == "mcmc") { + return []([[maybe_unused]] TRNG rng, ui32 n, [[maybe_unused]] double mu, [[maybe_unused]] double sigma) { + Cout << "================================= METRICS ================================\n"; + auto sampledDegrees = GenerateLogNormalDegrees(n, mu, sigma); + Cout << "sampled degrees: " << joinVector(sampledDegrees) << "\n"; + + auto graphicDegrees = MakeGraphicConnected(initialDegrees); + Cout << "graphic degrees: " << joinVector(graphicDegrees) << "\n"; + + auto initialGraph = ConstructGraphHavelHakimi(graphicDegrees); + return initialGraph; + }; + } + + if (topologyName == "chung-lu") { + return []([[maybe_unused]] TRNG rng, ui32 n, [[maybe_unused]] double mu, [[maybe_unused]] double sigma) { + Cout << "================================= METRICS ================================\n"; + auto initialDegrees = GenerateLogNormalDegrees(n, mu, sigma); + Cout << "initial degrees: " << joinVector(initialDegrees) << "\n"; + + auto initialGraph = GenerateRandomChungLuGraph(rng, initialDegrees); + return initialGraph; + }; + } + + throw std::runtime_error("Unknown topology: '" + topologyName + "'"); + } + + void RunBenches(TTestContext &ctx, TBenchmarkConfig config, TArgs args) { + std::string resultType = args.GetStringOrDefault("result", "SE"); + + ui64 generationRepeats = args.GetArgOrDefault("gen", "1").GetValue(); + ui64 repeats = args.GetArgOrDefault("repeats", "1").GetValue(); + + std::string topologyName = args.GetStringOrDefault("type", "star"); + auto generateTopology = GetTopology(topologyName); + + std::string headerAggregate = "idx,N,alpha,theta,sigma,mu," + BoxPlotCSVHeader(); + std::string header = "idx,seed,graph_name,aggregate_" + headerAggregate; + + ui32 idx = 0; + ui32 aggregateIdx = 0; - ui64 mcmcSteps = args.GetArgOrDefault("mcmcSteps", "100").GetValue(); for (double alpha : args.GetArgOrDefault("alpha", "0.5")) { for (double theta : args.GetArgOrDefault("theta", "1.0")) { for (double sigma : args.GetArgOrDefault("sigma", "0.5")) { for (double mu : args.GetArgOrDefault("mu", "1.0")) { - for (ui64 n : args.GetArg("N")) { - std::map> cummulative; - - ui64 generationRepeats = args.GetArgOrDefault("gen", "1").GetValue(); + for (ui64 n : args.GetArg("N")) { + std::stringstream commonParams; + commonParams << (aggregateIdx ++) + << "," << n + << "," << alpha << "," << theta + << "," << sigma << "," << mu; + + std::map> aggregate; for (ui64 j = 0; j < generationRepeats; ++ j) { Cout << "\n\n\n"; auto initialGraph = generateTopology(ctx.RNG, n, mu, sigma); - ui64 repeats = args.GetArgOrDefault("repeats", "1").GetValue(); for (ui64 i = 0; i < repeats; ++ i) { - Cout << "\n\n"; - Cout << "Reproduce: 'N=" << n << "; alpha=" << alpha << "; theta=" - << theta << "; sigma=" << sigma << "; mu=" << mu - << "; seed=" << ctx.RNG.Serialize() << "'\n"; - ui64 seed = ctx.RNG.Serialize(); + Cout << "\n\nReproduce: ya make -r -ttt --test-param TOPOLOGY='"; + Cout << "type=" << topologyName << "; " + << "N=" << n << "; " + << "alpha=" << alpha << "; " + << "theta=" << theta << "; " + << "sigma=" << sigma << "; " + << "mu=" << mu << "; " + << "seed=" << ctx.RNG.Serialize() << "'\n"; + + TRelationGraph graph = initialGraph; if (i != 0) { - MCMCRandomize(ctx.RNG, graph, mcmcSteps); + MCMCRandomize(ctx.RNG, graph); } graph.SetupKeysPitmanYor(ctx.RNG, TPitmanYorConfig{.Alpha = alpha, .Theta = theta}); @@ -460,13 +524,11 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { goto stop; } - AccumulateAllStats(cummulative, *result); + AccumulateAllStats(aggregate, *result); + std::string graphName = WriteGraph(ctx, idx, graph); std::stringstream params; - params << n << "," << (globalNum ++) << "," << i - << "," << alpha << "," << theta << "," << sigma - << "," << mu << "," << repeats << "," << seed << ","; - + params << idx ++ << "," << seed << "," << graphName << "," << commonParams.str(); WriteAllStats(ctx, "", header, params.str(), *result); } catch (std::exception &exc) { Cout << "Skipped run: " << exc.what() << "\n"; @@ -475,13 +537,8 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { } } - std::stringstream params; - params << n << "," << (cummulativeNum ++) - << "," << alpha << "," << theta - << "," << sigma << "," << mu - << "," << generationRepeats << ","; - WriteAllStats(ctx, "cummulative-", headerCummulative, params.str(), cummulative); + WriteAllStats(ctx, "aggregate-", headerAggregate, commonParams.str(), aggregate); } stop:; } @@ -490,100 +547,22 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { } } - template - void RunTrivialTopology(TTestContext &ctx, TBenchmarkConfig config, TArgs args) { - // if (ctx.OS) { - // (*ctx.OS) << "N,alpha,theta,seed,"; - // DumpBoxPlotCSVHeader(*ctx.OS); - // (*ctx.OS) << "\n"; - // } - - std::string resultType = "SE"; - if (args.HasArg("result")) { - resultType = args.GetString("result"); - } - - for (ui64 n : args.GetArg("N")) { - for (double alpha : args.GetArgOrDefault("alpha", "0.5")) { - for (double theta : args.GetArgOrDefault("theta", "1.0")) { - ui64 seed = ctx.RNG.Serialize(); - - Cout << "Reproduce: 'N=" << n << "; alpha=" << alpha << "; theta=" - << theta << "; seed=" << seed << "'\n"; - - TRelationGraph graph = TrivialTopology(ctx.RNG, n); - graph.SetupKeysPitmanYor(ctx.RNG, TPitmanYorConfig{.Alpha = alpha, .Theta = theta}); - - auto result = BenchmarkShuffleEliminationOnTopology(config, ctx.Session, resultType, graph); - if (!result) { - goto stop; - } - // if (ctx.OS) { - // (*ctx.OS) << n << "," << alpha << "," << theta << "," << seed << ","; - // DumpBoxPlotToCSV(*ctx.OS, result->GetStatistics()); - // (*ctx.OS) << "\n"; - // } - } - } - } - stop:; - } Y_UNIT_TEST(Benchmark) { TArgs args{GetTestParam("TOPOLOGY")}; if (!args.HasArg("N")) { + // prevent this test from launching non-interactively return; } - std::string topology = "star"; - if (args.HasArg("type")) { - topology = args.GetString("type"); - } - auto config = GetBenchmarkConfig(args); DumpBenchmarkConfig(Cout, config); - uint64_t state = 0; - if (args.HasArg("seed")) { - state = args.GetArg("seed").GetValue(); - } - + uint64_t state = args.GetArgOrDefault("seed", "0").GetValue(); TTestContext ctx = CreateTestContext(args, state, GetTestParam("SAVE_DIR")); - auto mcmc = [&]([[maybe_unused]] TRNG rng, ui32 n, double mu, double sigma) { - Cout << "================================= METRICS ================================\n"; - auto initialDegrees = GenerateLogNormalDegrees(n, mu, sigma); - Cout << "initial degrees: " << joinVector(initialDegrees) << "\n"; - - auto fixedDegrees = MakeGraphicConnected(initialDegrees); - Cout << "fixed degrees: " << joinVector(initialDegrees) << "\n"; - - auto initialGraph = ConstructGraphHavelHakimi(fixedDegrees); - return initialGraph; - }; - - auto star = [&]([[maybe_unused]] TRNG rng, ui32 n, [[maybe_unused]] double mu, [[maybe_unused]] double sigma) { - return GenerateStar(rng, n); - }; - - auto path = [&]([[maybe_unused]] TRNG rng, ui32 n, [[maybe_unused]] double mu, [[maybe_unused]] double sigma) { - return GenerateLine(rng, n); - }; - - auto clique = [&]([[maybe_unused]] TRNG rng, ui32 n, [[maybe_unused]] double mu, [[maybe_unused]] double sigma) { - return GenerateLine(rng, n); - }; - - if (topology == "mcmc") { - RunBenches(ctx, config, args, mcmc); - } else if (topology == "star") { - RunBenches(ctx, config, args, star); - } else if (topology == "path") { - RunBenches(ctx, config, args, path); - } else if (topology == "clique") { - RunBenches(ctx, config, args, clique); - } + RunBenches(ctx, config, args); } From 2859d012ddaee4802b0361278f79e40be4e92c12 Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Wed, 5 Nov 2025 02:40:40 +0000 Subject: [PATCH 32/43] Large refactoring of kqp_benches --- ydb/core/kqp/ut/common/kqp_benches.h | 508 ++++++------------ ydb/core/kqp/ut/common/ya.make | 1 + ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp | 213 +++++--- 3 files changed, 293 insertions(+), 429 deletions(-) diff --git a/ydb/core/kqp/ut/common/kqp_benches.h b/ydb/core/kqp/ut/common/kqp_benches.h index d9ac9fcd5930..6245285f5087 100644 --- a/ydb/core/kqp/ut/common/kqp_benches.h +++ b/ydb/core/kqp/ut/common/kqp_benches.h @@ -1,12 +1,18 @@ #pragma once +#include +#include +#include +#include #include +#include #include #include #include #include #include +#include #include #include @@ -24,416 +30,234 @@ ui64 MeasureTimeNanos(Lambda &&lambda) { return Nsc::duration_cast(TClock::now() - start).count(); } -class TimeFormatter { -public: - static std::string Format(uint64_t valueNs, ui64 uncertaintyNs) { - auto [unit, scale] = SelectUnit(valueNs); - - double scaledValue = valueNs / scale; - double scaledUncertainty = uncertaintyNs / scale; - - int decimalPlaces = GetDecimalPlaces(scaledUncertainty); - - std::ostringstream oss; - oss << std::fixed << std::setprecision(decimalPlaces); - oss << scaledValue << " " << unit << " ± " << scaledUncertainty << " " << unit; - - return oss.str(); - } - - static std::string Format(uint64_t valueNs) { - auto [unit, scale] = SelectUnit(valueNs); - double scaledValue = valueNs / scale; - - int decimalPlaces = GetDecimalPlacesForValue(scaledValue); - - std::ostringstream oss; - oss << std::fixed << std::setprecision(decimalPlaces); - oss << scaledValue << " " << unit; - - return oss.str(); - } - -private: - static std::pair SelectUnit(ui64 nanoseconds) { - if (nanoseconds >= 1'000'000'000) { - return {"s", 1e9}; - } else if (nanoseconds >= 1'000'000) { - return {"ms", 1e6}; - } else if (nanoseconds >= 1'000) { - return {"μs", 1e3}; - } else { - return {"ns", 1.0}; - } - } - static int GetDecimalPlaces(double scaledUncertainty) { - if (scaledUncertainty < 0.01) return 2; - - double log_val = std::log10(scaledUncertainty); - int magnitude = static_cast(std::floor(log_val)); - - if (scaledUncertainty >= 100.0) { - return 0; - } else if (scaledUncertainty >= 10.0) { - return 1; - } else if (scaledUncertainty >= 1.0) { - return 1; - } else { - return std::max(0, -magnitude + 1); - } - } - - static int GetDecimalPlacesForValue(double scaledValue) { - if (scaledValue < 0.01) return 4; - - double absValue = std::abs(scaledValue); - - if (absValue >= 1000.0) { - return 0; - } else if (absValue >= 100.0) { - return 1; - } else if (absValue >= 10.0) { - return 2; - } else if (absValue >= 1.0) { - return 2; - } else if (absValue >= 0.1) { - return 3; - } else { - return 4; - } - } - -}; - -template -struct TStatistics { +struct TComputedStatistics { ui64 N; - - TValue Min, Max; + double Min, Max; double Median; double MAD; - double Mean; - double Stdev; - double Q1; double Q3; - double IQR; - std::vector Outliers; + + double Mean; + double Stdev; + + static std::string GetCSVHeader(); + void ToCSV(IOutputStream &os) const; }; -void DumpTimeStatistics(TStatistics stats, IOutputStream &OS) { - OS << "Median = " << TimeFormatter::Format(stats.Median, stats.MAD) << " (MAD)\n"; - OS << "N = " << stats.N << "\n"; - OS << "Min, Max = [ " << TimeFormatter::Format(stats.Min) << ", " << TimeFormatter::Format(stats.Max) << " ]\n"; +double CalculatePercentile(const std::vector& data, double percentile); +double CalculateMedian(std::vector& data); +double CalculateMAD(const std::vector& data, double median, std::vector& storage); +double CalculateMean(const std::vector& data); +double CalculateSampleStdDev(const std::vector& data, double mean); - OS << "Mean = " << TimeFormatter::Format(stats.Mean, stats.Stdev) << " (Std. dev.)\n"; -} -// TValue is arithmetic and can be compared with itself, added to itself, and dividable by floats -// TValue is assumed to be cheap to pass by value -template -class TRunningStatistics { +class TStatistics { public: - TRunningStatistics() - : MaxHeap_() - , MinHeap_() - , N_(0) - , Total_(0) - , Min_(std::nullopt) - , Max_(std::nullopt) + TStatistics(std::vector samples, double min, double max) + : Samples_(std::move(samples)) + , Min_(min) + , Max_(max) { } - void AddValue(TValue num) { - Total_ += num; - ++ N_; + TComputedStatistics ComputeStatistics() const; - if (!Min_) { - Min_ = num; - } else { - Min_ = Min(*Min_, num); - } + ui64 GetN() const { + return Samples_.size(); + } - if (!Max_) { - Max_ = num; - } else { - Max_ = Max(*Max_, num); - }; + double GetMin() const { + return Min_; + } - if (MaxHeap_.empty() || num <= MaxHeap_[0]) { - MaxHeap_.push_back(num); - std::push_heap(MaxHeap_.begin(), MaxHeap_.end()); - } else { - MinHeap_.push_back(num); - std::push_heap(MinHeap_.begin(), MinHeap_.end(), std::greater{}); - } + double GetMax() const { + return Max_; + } - if (MaxHeap_.size() > MinHeap_.size() + 1) { - // MinHeap_.push(MaxHeap_.top()); - MinHeap_.push_back(MaxHeap_[0]); - std::push_heap(MinHeap_.begin(), MinHeap_.end(), std::greater{}); - - // MaxHeap_.pop(); - std::pop_heap(MaxHeap_.begin(), MaxHeap_.end()); - MaxHeap_.pop_back(); - } else if (MinHeap_.size() > MaxHeap_.size()) { - // MaxHeap_.push(MinHeap_.top()); - MaxHeap_.push_back(MinHeap_[0]); - std::push_heap(MaxHeap_.begin(), MaxHeap_.end()); - - // MinHeap_.pop(); - std::pop_heap(MinHeap_.begin(), MinHeap_.end(), std::greater{}); - MinHeap_.pop_back(); - } + void Merge(TStatistics other) { + Samples_.insert(Samples_.end(), other.Samples_.begin(), other.Samples_.end()); + Min_ = std::min(Min_, other.Min_); + Max_ = std::min(Max_, other.Max_); } - double GetMean() const { - assert(N_ != 0); + // Operations on two random values, very expensive. + // Each operation produces cartesian product of all possibilities if + // it's small enough or fallbacks to Monte Carlo methods if not. + TStatistics operator-(const TStatistics &rhs) const; + TStatistics operator+(const TStatistics &rhs) const; + TStatistics operator*(const TStatistics &rhs) const; + TStatistics operator/(const TStatistics &rhs) const; + + // Only gives precisely correct min & max if you map with monotonic function + template + TStatistics Map(TMapLambda map) { + std::vector samples; + double min = std::numeric_limits::max(); + double max = std::numeric_limits::min(); + + for (double value : Samples_) { + double newValue = map(value); + samples.push_back(newValue); + + min = std::min(newValue, min); + max = std::max(newValue, max); + } - return Total_ / static_cast(N_); + double candidates[] = { min, max, map(Min_), map(Max_) }; + return { + std::move(samples), + *std::min_element(candidates, candidates + Y_ARRAY_SIZE(candidates)), + *std::max_element(candidates, candidates + Y_ARRAY_SIZE(candidates)) + }; } - double CalculateStdDev() const { - assert(N_ != 0); + template + TStatistics Filter(TFilterLambda &&filter) const { + std::vector samples; - if (N_ == 1) { - return 0; - } + // That's an upper bound since this function can only remove elements. + // This resize can be memory inefficient, but is likely faster. + samples.reserve(Samples_.size()); - auto mean = GetMean(); + double min = std::numeric_limits::max(); + double max = std::numeric_limits::min(); - double deviationSquaresSum = 0; - auto addValue = [&](TValue value) { - double deviation = abs(value - mean); - deviationSquaresSum += deviation * deviation; - }; + for (double value : Samples_) { + if (!filter(value)) { + continue; + } - for (TValue value : MaxHeap_) { - addValue(value); - } + samples.push_back(value); - for (TValue value : MinHeap_) { - addValue(value); + min = std::min(value, min); + max = std::max(value, max); } - return std::sqrt(deviationSquaresSum / (N_ - 1)); - } - - double GetMedian() const { - assert(!MaxHeap_.empty() || !MinHeap_.empty()); + double globalMin = filter(Min_) ? Min_ : min; + double globalMax = filter(Max_) ? Max_ : max; - if (MaxHeap_.size() > MinHeap_.size()) { - return MaxHeap_[0]; - } else { - return (MaxHeap_[0] + MinHeap_[0]) / 2.0; - } + return {std::move(samples), globalMin, globalMax}; } - double CalculateMAD() const { - assert(!MaxHeap_.empty() || !MinHeap_.empty()); - std::vector deviation; - double median = GetMedian(); +private: + mutable std::vector Samples_; + double Min_; + double Max_; - IterateElements([&](TValue value) { - deviation.push_back(std::abs(value - median)); - }); + // This lets us operate on two random values, it computes distribution of values after arbitrary operation + template + std::vector + Combine(const TStatistics& rhs, TCombiner&& combine, ui64 maxSamples = 1e8) const { - std::sort(deviation.begin(), deviation.end()); - return (deviation[deviation.size() / 2] + deviation[(deviation.size() - 1) / 2]) / 2.0; + std::vector combined; - } + ui64 cartesianProductSize = Samples_.size() * rhs.Samples_.size(); - TValue GetMin() const { - assert(Min_); - return *Min_; - } - - TValue GetMax() const { - assert(Max_); - return *Max_; - } + if (cartesianProductSize < maxSamples) { + combined.reserve(cartesianProductSize); - TValue GetTotal() const { - return Total_; - } - - ui32 GetN() const { - return N_; - } + for (ui64 i = 0; i < Samples_.size(); ++ i) { + for (ui64 j = 0; j < rhs.Samples_.size(); ++ j) { + combined.push_back(combine(Samples_[i], rhs.Samples_[j])); + } + } - TStatistics GetStatistics() const { - auto elements = CollectElements(); - std::sort(elements.begin(), elements.end()); + return combined; + } - double q1 = CalculateQuartile(elements, 0.25); - double q3 = CalculateQuartile(elements, 0.75); - double iqr = q3 - q1; + // Fallback to Monte Carlo if the sample size is too big: + std::random_device randomDevice; + std::mt19937 rng(/*seed=*/randomDevice()); - double median = GetMedian(); + std::uniform_int_distribution<> distributionLHSIdx(0, Samples_.size() - 1); + std::uniform_int_distribution<> distributionRHSIdx(0, rhs.Samples_.size() - 1); - const double multiplier = 1.5; - double lowerFence = q1 - multiplier * iqr; - double upperFence = q3 + multiplier * iqr; + for (ui64 repeatIdx = 0; repeatIdx < maxSamples; ++ repeatIdx) { + ui64 i = distributionLHSIdx(rng); + ui64 j = distributionRHSIdx(rng); - std::vector outliers; - for (TValue value : elements) { - if (value < lowerFence || value > upperFence) { - outliers.push_back(value); - } + combined.push_back(combine(Samples_[i], rhs.Samples_[j])); } - return { - .N = GetN(), - .Min = GetMin(), - .Max = GetMax(), - .Median = median, - .MAD = CalculateMAD(), - .Mean = GetMean(), - .Stdev = CalculateStdDev(), - .Q1 = q1, - .Q3 = q3, - .IQR = iqr, - .Outliers = outliers - }; + return combined; } - void AddValues(const TRunningStatistics& other) { - TRunningStatistics values; - other.IterateElements([&](TValue value) { - AddValue(value); - }); - } +}; - std::vector CollectElements() const { - std::vector values; - IterateElements([&](TValue value) { - values.push_back(value); - }); - return values; +class TRunningStatistics { +public: + TRunningStatistics() + : MinHeap_() + , MaxHeap_() + , Samples_() + , MADStorage_() + , Total_(0) + , Min_(std::nullopt) + , Max_(std::nullopt) + { } - template - TRunningStatistics Cast() const { - if constexpr (std::is_same_v) { - return *this; - } else { - TRunningStatistics stats; - IterateElements([&](TValue lhs) { - stats.AddValue(static_cast(lhs)); - }); - - return stats; - } - } + void AddValue(double num); - template - TRunningStatistics Reject(Lambda &&shouldReject) const { - TRunningStatistics stats; - IterateElements([&](TValue value) { - if (!shouldReject(value)) { - stats.AddValue(value); - } - }); + double GetMedian() const; + double CalculateMAD() const; - return stats; + double GetTotal() const { + return Total_; } - TRunningStatistics operator-(TRunningStatistics rhsStats) const { - TRunningStatistics stats; - IterateElements([&](TValue lhs) { - rhsStats.IterateElements([&](TValue rhs) { - stats.AddValue((i64) lhs - (i64) rhs); - }); - }); - - return stats; + ui32 GetN() const { + return Samples_.size(); } - TRunningStatistics operator/(TRunningStatistics rhsStats) const { - TRunningStatistics stats; - IterateElements([&](TValue lhs) { - rhsStats.IterateElements([&](TValue rhs) { - stats.AddValue((double) lhs / (double) rhs); - }); - }); - - return stats; + double GetMin() const { + assert(Min_); + return *Min_; } -private: - std::vector MaxHeap_; - std::vector MinHeap_; - - // std::priority_queue> MaxHeap_; - // std::priority_queue, std::greater> MinHeap_; - - ui32 N_; - TValue Total_; + double GetMax() const { + assert(Max_); + return *Max_; + } - std::optional Min_; - std::optional Max_; + TStatistics GetStatistics() { + assert(GetN() > 0); + return {Samples_, *Min_, *Max_}; + } private: - template - void IterateElements(Lambda &&lambda) const { - for (TValue value : MaxHeap_) { - lambda(value); - } - - for (TValue value : MinHeap_) { - lambda(value); - } - } + std::priority_queue, std::greater> MinHeap_; + std::priority_queue> MaxHeap_; - static double CalculateQuartile(const std::vector& data, double percentile) { // TODO: rename - assert(percentile >= 0.0 && percentile <= 1.0); - assert(std::is_sorted(data.begin(), data.end())); + std::vector Samples_; - double position = percentile * (data.size() - 1); - int lowerIndex = floor(position); - int upperIndex = ceil(position); + // This vector is used to not allocate memory every time + // we are requested to compute MAD and need to calculate diviations + mutable std::vector MADStorage_; - if (lowerIndex == upperIndex) { - return data[lowerIndex]; - } + double Total_; - double weight = position - lowerIndex; - return data[lowerIndex] * (1 - weight) + data[upperIndex] * weight; - } + std::optional Min_; + std::optional Max_; }; -template -static std::string joinVector(const std::vector &data, std::string delimeter = ";") { - if (data.empty()) { - return ""; - } - - std::string str; - str += std::to_string(data[0]); - - for (ui32 i = 1; i < data.size(); ++ i) { - str += delimeter + std::to_string(data[i]); - } - - return str; -} -std::string BoxPlotCSVHeader() { - return "median,Q1,Q3,IQR,MAD,min,max"; -} +class TimeFormatter { +public: + static std::string Format(uint64_t valueNs, ui64 uncertaintyNs); + static std::string Format(uint64_t valueNs); +}; -template -void DumpBoxPlotToCSV(IOutputStream &os, TStatistics stat) { - os << stat.Median << "," << stat.Q1 << "," << stat.Q3 << "," << stat.IQR << "," << stat.MAD << "," << stat.Min << "," << stat.Max; -} +void DumpTimeStatistics(const TComputedStatistics &stats, IOutputStream &OS); struct TRepeatedTestConfig { @@ -443,25 +267,29 @@ struct TRepeatedTestConfig { }; template -std::optional> RepeatedTest(TRepeatedTestConfig config, ui64 singleRunTimeout, double thresholdMAD, TLambda &&lambda) { +std::optional RepeatedTest(TRepeatedTestConfig config, ui64 singleRunTimeout, double thresholdMAD, TLambda &&lambda) { assert(config.MinRepeats >= 1); - TRunningStatistics stats; + TRunningStatistics stats; do { bool hasTimedOut = false; ui64 ellapsedTime = MeasureTimeNanos([&]() { hasTimedOut = !lambda(); }); - if (hasTimedOut || ellapsedTime > singleRunTimeout) { + if (hasTimedOut) { return std::nullopt; } + if (ellapsedTime > singleRunTimeout) { + break; + } + stats.AddValue(ellapsedTime); } while ((stats.GetN() < config.MaxRepeats) && (stats.GetN() < config.MinRepeats || - (stats.GetTotal() + stats.GetMedian() <= config.Timeout && + (stats.GetTotal() + stats.GetMedian() < config.Timeout && stats.CalculateMAD() / stats.GetMedian() > thresholdMAD))); - return stats; + return std::move(stats).GetStatistics(); } @@ -473,7 +301,7 @@ struct TBenchmarkConfig { }; template -std::optional> Benchmark(TBenchmarkConfig config, Lambda &&lambda) { +std::optional Benchmark(TBenchmarkConfig config, Lambda &&lambda) { if (config.Warmup.MaxRepeats != 0) { auto warmupStats = RepeatedTest(config.Warmup, config.SingleRunTimeout, config.MADThreshold, lambda); if (!warmupStats) { diff --git a/ydb/core/kqp/ut/common/ya.make b/ydb/core/kqp/ut/common/ya.make index 64e07aed5aa0..ec6556b78113 100644 --- a/ydb/core/kqp/ut/common/ya.make +++ b/ydb/core/kqp/ut/common/ya.make @@ -12,6 +12,7 @@ SRCS( math_udf.cpp unicode_udf.cpp digest_udf.cpp + kqp_benches.cpp ) PEERDIR( diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp index 5b82f871b594..eeb7ad71e8da 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp @@ -79,9 +79,9 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { return queryWithShuffleElimination; } - std::optional> BenchmarkExplain(TBenchmarkConfig config, NYdb::NQuery::TSession session, const TString& query) { + std::optional BenchmarkExplain(TBenchmarkConfig config, NYdb::NQuery::TSession session, const TString& query) { std::optional savedPlan = std::nullopt; - auto stats = Benchmark(config, [&]() -> bool { + std::optional stats = Benchmark(config, [&]() -> bool { auto plan = ExplainQuery(session, query); if (!savedPlan) { savedPlan = plan; @@ -99,81 +99,77 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { JustPrintPlan(*savedPlan); Cout << "--------------------------------------------------------------------------\n"; - DumpTimeStatistics(stats->GetStatistics(), Cout); + DumpTimeStatistics(stats->ComputeStatistics(), Cout); return stats; } - std::optional>> BenchmarkShuffleElimination(TBenchmarkConfig config, NYdb::NQuery::TSession session, std::string resultType, const TString& query) { - std::map> results; + std::optional> BenchmarkShuffleElimination(TBenchmarkConfig config, NYdb::NQuery::TSession session, std::string resultType, const TString& query) { + std::map results; - std::optional> withoutCBO; + std::optional withoutCBO; if (resultType.contains("0")) { Cout << "--------------------------------- W/O CBO --------------------------------\n"; - withoutCBO = BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/false, /*optLevel=*/0)); - results["0"] = withoutCBO->Cast(); + withoutCBO = BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/false, /*optLevel=*/0)); if (!withoutCBO) { return std::nullopt; } - if (resultType == "0") { + results.emplace("0", *withoutCBO); + if (withoutCBO->GetMax() > config.SingleRunTimeout || resultType == "0") { return results; } } - std::optional> withoutShuffleElimination; + std::optional withoutShuffleElimination; if (resultType.contains("CBO")) { Cout << "--------------------------------- CBO-SE ---------------------------------\n"; withoutShuffleElimination = BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/false, /*optLevel=*/2)); - results["CBO"] = withoutShuffleElimination->Cast(); if (resultType.contains("0")) { - results["CBO-0"] = (*withoutShuffleElimination - *withoutCBO).Cast(); + results.emplace("CBO-0", (*withoutShuffleElimination - *withoutCBO).Filter([](double value) { return value >= 0; })); } if (!withoutShuffleElimination) { return std::nullopt; } - if (resultType == "CBO" || resultType == "CBO-0") { + results.emplace("CBO", *withoutShuffleElimination); + if (withoutShuffleElimination->GetMax() > config.SingleRunTimeout || resultType == "CBO" || resultType == "CBO-0") { return results; } } - std::optional> withShuffleElimination; + std::optional withShuffleElimination; if (resultType.contains("SE")) { Cout << "--------------------------------- CBO+SE ---------------------------------\n"; withShuffleElimination = BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/true, /*optLevel=*/2)); - results["SE"] = withShuffleElimination->Cast(); if (resultType.contains("0")) { - results["SE-0"] = (*withShuffleElimination - *withoutCBO).Cast(); + results.emplace("SE-0", (*withShuffleElimination - *withoutCBO).Filter([](double value) { return value >= 0; })); } if (!withShuffleElimination) { return std::nullopt; } - if (resultType == "SE" || resultType == "SE-0") { + results.emplace("SE", *withShuffleElimination); + if (withShuffleElimination->GetMax() > config.SingleRunTimeout || resultType == "SE" || resultType == "SE-0") { return results; } } Cout << "--------------------------------------------------------------------------\n"; - results["SE-div-CBO"] = *withShuffleElimination / *withoutShuffleElimination; + results.emplace("SE-div-CBO", *withShuffleElimination / *withoutShuffleElimination); if (resultType.contains("0")) { - if (withShuffleElimination->GetMedian() > withoutShuffleElimination->GetMedian() || - withShuffleElimination->GetMedian() > withoutCBO->GetMedian()) { - - auto adjustedWithTime = (*withShuffleElimination - *withoutCBO).Reject([](i64 value) { return value < 0; }); - auto adjustedWithoutTime = (*withoutShuffleElimination - *withoutCBO).Reject([](i64 value) { return value < 0; }); - - auto adjustedRatio = adjustedWithTime / adjustedWithoutTime; - adjustedRatio = adjustedRatio.Reject([](double value) { return value < 1.0; }); + auto& adjustedWithTime = results.at("SE-0"); + auto& adjustedWithoutTime = results.at("CBO-0"); - results["SE-0-div-CBO-0"] = adjustedRatio; + if (adjustedWithTime.ComputeStatistics().Median > adjustedWithoutTime.ComputeStatistics().Median) { + auto adjustedRatio = (adjustedWithTime / adjustedWithoutTime).Filter([](double value) { return value >= 1.0; }); + results.emplace("SE-0-div-CBO-0", adjustedRatio); } } @@ -202,7 +198,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { return std::make_unique(serverSettings); } - std::optional>> + std::optional> BenchmarkShuffleEliminationOnTopology(TBenchmarkConfig config, NYdb::NQuery::TSession session, std::string resultType, TRelationGraph graph) { Cout << "================================= CREATE =================================\n"; graph.DumpGraph(Cout); @@ -218,26 +214,21 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { return std::nullopt; } - Cout << "================================= BENCHMARK ==============================\n"; - TString query = graph.MakeQuery(); - Cout << query; - - Cout << "================================= FINALIZE ===============================\n"; - auto deletionQuery = graph.GetSchema().MakeDropQuery(); - Cout << deletionQuery; + Y_SCOPE_EXIT(&) { + Cout << "================================= FINALIZE ===============================\n"; + auto deletionQuery = graph.GetSchema().MakeDropQuery(); + Cout << deletionQuery; - try { - auto resultTime = BenchmarkShuffleElimination(config, session, resultType, query); - ExecuteQuery(session, deletionQuery); + bool deletionSucceeded = ExecuteQuery(session, deletionQuery); + UNIT_ASSERT_C(deletionSucceeded, "Table deletion timeouted, can't proceed!"); Cout << "==========================================================================\n"; + }; - return resultTime; - } catch (std::exception &exc) { - ExecuteQuery(session, deletionQuery); - Cout << "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! FAILED !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"; + Cout << "================================= BENCHMARK ==============================\n"; + TString query = graph.MakeQuery(); + Cout << query; - throw std::runtime_error(std::string("Benchmark failed with '") + exc.what() + "'"); - } + return BenchmarkShuffleElimination(config, session, resultType, query); } @@ -375,7 +366,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { void WriteAllStats(TTestContext &ctx, const std::string& prefix, const std::string& header, const std::string& params, - const std::map>& stats) { + const std::map& stats) { if (ctx.OutputDir.empty()) { return; @@ -385,7 +376,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { std::filesystem::create_directories(ctx.OutputDir); } - for (const auto &[key, stats]: stats) { + for (const auto &[key, stat]: stats) { std::string name = prefix + key; if (!ctx.Streams.contains(name)) { @@ -397,19 +388,19 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { auto &os = ctx.Streams.find(name)->second; os << params << ","; - DumpBoxPlotToCSV(os, stats.GetStatistics()); + stat.ComputeStatistics().ToCSV(os); os << "\n"; } } - void AccumulateAllStats(std::map>& cummulative, - const std::map>& stats) { + void AccumulateAllStats(std::map& cummulative, + const std::map& stats) { for (auto &[key, stat] : stats) { if (!cummulative.contains(key)) { cummulative.emplace(key, stat); } - cummulative.find(key)->second.AddValues(stat); + cummulative.find(key)->second.Merge(stat); } } @@ -443,10 +434,10 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { return []([[maybe_unused]] TRNG rng, ui32 n, [[maybe_unused]] double mu, [[maybe_unused]] double sigma) { Cout << "================================= METRICS ================================\n"; auto sampledDegrees = GenerateLogNormalDegrees(n, mu, sigma); - Cout << "sampled degrees: " << joinVector(sampledDegrees) << "\n"; + Cout << "sampled degrees: " << JoinSeq(", ", sampledDegrees) << "\n"; - auto graphicDegrees = MakeGraphicConnected(initialDegrees); - Cout << "graphic degrees: " << joinVector(graphicDegrees) << "\n"; + auto graphicDegrees = MakeGraphicConnected(sampledDegrees); + Cout << "graphic degrees: " << JoinSeq(", ", graphicDegrees) << "\n"; auto initialGraph = ConstructGraphHavelHakimi(graphicDegrees); return initialGraph; @@ -457,7 +448,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { return []([[maybe_unused]] TRNG rng, ui32 n, [[maybe_unused]] double mu, [[maybe_unused]] double sigma) { Cout << "================================= METRICS ================================\n"; auto initialDegrees = GenerateLogNormalDegrees(n, mu, sigma); - Cout << "initial degrees: " << joinVector(initialDegrees) << "\n"; + Cout << "initial degrees: " << JoinSeq(", ", initialDegrees) << "\n"; auto initialGraph = GenerateRandomChungLuGraph(rng, initialDegrees); return initialGraph; @@ -476,7 +467,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { std::string topologyName = args.GetStringOrDefault("type", "star"); auto generateTopology = GetTopology(topologyName); - std::string headerAggregate = "idx,N,alpha,theta,sigma,mu," + BoxPlotCSVHeader(); + std::string headerAggregate = "idx,N,alpha,theta,sigma,mu," + TComputedStatistics::GetCSVHeader(); std::string header = "idx,seed,graph_name,aggregate_" + headerAggregate; ui32 idx = 0; @@ -493,7 +484,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { << "," << alpha << "," << theta << "," << sigma << "," << mu; - std::map> aggregate; + std::map aggregate; for (ui64 j = 0; j < generationRepeats; ++ j) { Cout << "\n\n\n"; auto initialGraph = generateTopology(ctx.RNG, n, mu, sigma); @@ -565,59 +556,103 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { RunBenches(ctx, config, args); } + void Check(TRunningStatistics& stats, ui32 n, double min, double max, double median, double mad, double q1, double q3, double iqr, double mean, double stdev) { + const double TOLERANCE = 0.0001; + + UNIT_ASSERT_EQUAL(stats.GetN(), n); + UNIT_ASSERT_DOUBLES_EQUAL(stats.GetMin(), min, TOLERANCE); + UNIT_ASSERT_DOUBLES_EQUAL(stats.GetMax(), max, TOLERANCE); + UNIT_ASSERT_DOUBLES_EQUAL(stats.GetMedian(), median, TOLERANCE); + UNIT_ASSERT_DOUBLES_EQUAL(stats.CalculateMAD(), mad, TOLERANCE); + + auto statistics = stats.GetStatistics(); + UNIT_ASSERT_EQUAL(statistics.GetN(), n); + UNIT_ASSERT_DOUBLES_EQUAL(statistics.GetMin(), min, TOLERANCE); + UNIT_ASSERT_DOUBLES_EQUAL(statistics.GetMax(), max, TOLERANCE); + + auto report = statistics.ComputeStatistics(); + UNIT_ASSERT_EQUAL(report.N, n); + UNIT_ASSERT_DOUBLES_EQUAL(report.Min, min, TOLERANCE); + UNIT_ASSERT_DOUBLES_EQUAL(report.Max, max, TOLERANCE); + UNIT_ASSERT_DOUBLES_EQUAL(report.Median, median, TOLERANCE); + UNIT_ASSERT_DOUBLES_EQUAL(report.MAD, mad, TOLERANCE); + UNIT_ASSERT_DOUBLES_EQUAL(report.Q1, q1, TOLERANCE); + UNIT_ASSERT_DOUBLES_EQUAL(report.Q3, q3, TOLERANCE); + UNIT_ASSERT_DOUBLES_EQUAL(report.IQR, iqr, TOLERANCE); + UNIT_ASSERT_DOUBLES_EQUAL(report.Mean, mean, TOLERANCE); + UNIT_ASSERT_DOUBLES_EQUAL(report.Stdev, stdev, TOLERANCE); + } - Y_UNIT_TEST(TRunningStatistics) { - TRunningStatistics stats; - stats.AddValue(1); - UNIT_ASSERT_EQUAL(stats.GetMedian(), 1); - UNIT_ASSERT_EQUAL(stats.GetMean(), 1 / 1.0); - UNIT_ASSERT_EQUAL(stats.GetN(), 1); + Y_UNIT_TEST(TStatistics) { + TRunningStatistics stats; stats.AddValue(1); - UNIT_ASSERT_EQUAL(stats.GetMedian(), (1 + 1) / 2.0); - UNIT_ASSERT_EQUAL(stats.GetMean(), 2 / 2.0); - UNIT_ASSERT_EQUAL(stats.GetN(), 2); + stats.AddValue(1); + Check(stats, 2, 1, 1, 1, 0, 1, 1, 0, 1, 0); stats.AddValue(1); - UNIT_ASSERT_EQUAL(stats.GetMedian(), 1); - UNIT_ASSERT_EQUAL(stats.GetMean(), 3 / 3.0); - UNIT_ASSERT_EQUAL(stats.GetN(), 3); + Check(stats, 3, 1, 1, 1, 0, 1, 1, 0, 1, 0); stats.AddValue(1); - UNIT_ASSERT_EQUAL(stats.GetMedian(), (1 + 1) / 2.0); - UNIT_ASSERT_EQUAL(stats.GetMean(), 4 / 4.0); - UNIT_ASSERT_EQUAL(stats.GetN(), 4); + Check(stats, 4, 1, 1, 1, 0, 1, 1, 0, 1, 0); stats.AddValue(3); - UNIT_ASSERT_EQUAL(stats.GetMedian(), 1); - UNIT_ASSERT_EQUAL(stats.GetMean(), 7 / 5.0); - UNIT_ASSERT_EQUAL(stats.GetN(), 5); + Check(stats, 5, 1, 3, 1, 0, 1, 1, 0, 1.4000, 0.8944); stats.AddValue(5); - UNIT_ASSERT_EQUAL(stats.GetMedian(), (1 + 1) / 2.0); - UNIT_ASSERT_EQUAL(stats.GetMean(), 12 / 6.0); - UNIT_ASSERT_EQUAL(stats.GetN(), 6); + Check(stats, 6, 1, 5, 1, 0, 1, 2.5000, 1.5000, 2, 1.6733); stats.AddValue(7); - UNIT_ASSERT_EQUAL(stats.GetMedian(), 1); - UNIT_ASSERT_EQUAL(stats.GetMean(), 19 / 7.0); - UNIT_ASSERT_EQUAL(stats.GetN(), 7); + Check(stats, 7, 1, 7, 1, 0, 1, 4, 3, 2.7143, 2.4300); stats.AddValue(7); - UNIT_ASSERT_EQUAL(stats.GetMedian(), (1 + 3) / 2.0); - UNIT_ASSERT_EQUAL(stats.GetMean(), 26 / 8.0); - UNIT_ASSERT_EQUAL(stats.GetN(), 8); + Check(stats, 8, 1, 7, 2, 1, 1, 5.5000, 4.5000, 3.2500, 2.7124); stats.AddValue(8); - UNIT_ASSERT_EQUAL(stats.GetMedian(), 3); - UNIT_ASSERT_EQUAL(stats.GetMean(), 34 / 9.0); - UNIT_ASSERT_EQUAL(stats.GetN(), 9); + Check(stats, 9, 1, 8, 3, 2, 1, 7, 6, 3.7778, 2.9907); stats.AddValue(100); - UNIT_ASSERT_EQUAL(stats.GetMedian(), (3 + 5) / 2.0); - UNIT_ASSERT_EQUAL(stats.GetMean(), 134 / 10.0); - UNIT_ASSERT_EQUAL(stats.GetN(), 10); + Check(stats, 10, 1, 100, 4, 3, 1, 7, 6, 13.4000, 30.5585); + + stats.AddValue(12); + Check(stats, 11, 1, 100, 5, 4, 1, 7.5000, 6.5000, 13.2727, 28.9934); + + stats.AddValue(15); + Check(stats, 12, 1, 100, 6, 5, 1, 9, 8, 13.4167, 27.6486); + + stats.AddValue(11); + Check(stats, 13, 1, 100, 7, 5, 1, 11, 10, 13.2308, 26.4800); + + stats.AddValue(18); + Check(stats, 14, 1, 100, 7, 5.5000, 1.5000, 11.7500, 10.2500, 13.5714, 25.4731); + + stats.AddValue(19); + Check(stats, 15, 1, 100, 7, 6, 2, 13.5000, 11.5000, 13.9333, 24.5865); + + stats.AddValue(14); + Check(stats, 16, 1, 100, 7.5000, 6.5000, 2.5000, 14.2500, 11.7500, 13.9375, 23.7528); + + stats.AddValue(21); + Check(stats, 17, 1, 100, 8, 7, 3, 15, 12, 14.3529, 23.0623); + + stats.AddValue(9); + Check(stats, 18, 1, 100, 8.5000, 6, 3.5000, 14.7500, 11.2500, 14.0556, 22.4092); + + stats.AddValue(25); + Check(stats, 19, 1, 100, 9, 6, 4, 16.5000, 12.5000, 14.6316, 21.9221); + + stats.AddValue(17); + Check(stats, 20, 1, 100, 10, 7, 4.5000, 17.2500, 12.7500, 14.7500, 21.3440); + + stats.AddValue(18); + Check(stats, 21, 1, 100, 11, 7, 5, 18, 13, 14.9048, 20.8156); + + stats.AddValue(10); + Check(stats, 22, 1, 100, 10.5000, 7, 5.5000, 17.7500, 12.2500, 14.6818, 20.3409); + + stats.AddValue(22); + Check(stats, 23, 1, 100, 11, 7, 6, 18, 12, 15, 19.9317); } } From 9fb08defdf1025ff6af1c355270411f9c9dade6f Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Wed, 5 Nov 2025 05:29:22 +0000 Subject: [PATCH 33/43] Fix reproducibility of topology-based benchmarks --- ydb/core/kqp/ut/common/kqp_serializable_rng.h | 8 +- .../ut/join/kqp_join_topology_generator.cpp | 19 +-- .../kqp/ut/join/kqp_join_topology_generator.h | 4 +- ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp | 140 +++++++++++++----- 4 files changed, 115 insertions(+), 56 deletions(-) diff --git a/ydb/core/kqp/ut/common/kqp_serializable_rng.h b/ydb/core/kqp/ut/common/kqp_serializable_rng.h index 9065007af491..466a49909b8b 100644 --- a/ydb/core/kqp/ut/common/kqp_serializable_rng.h +++ b/ydb/core/kqp/ut/common/kqp_serializable_rng.h @@ -26,7 +26,7 @@ class TSerializableMT19937 { , Counter_(0) { } - + uint64_t Serialize() const { return (static_cast(Seed_) << 32ULL) | static_cast(Counter_); } @@ -39,6 +39,12 @@ class TSerializableMT19937 { Engine_.discard(Counter_); } + void Forward(uint32_t counter) { + assert(counter >= GetCounter()); + ui32 difference = counter - GetCounter(); + discard(difference); + } + uint32_t GetCounter() const { return Counter_; } diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp index a737e71fc7bd..ec4c96d41f4e 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp @@ -419,23 +419,17 @@ void NormalizeProbabilities(std::vector& probabilities) { } } -std::vector SampleFromPMF(const std::vector& probabilities, - int numVertices, int minDegree) { - std::random_device randomDevice; - std::mt19937 generator(randomDevice()); - std::discrete_distribution distribution(probabilities.begin(), - probabilities.end()); +std::vector SampleFromPMF(TRNG& rng, const std::vector& probabilities, int numVertices, int minDegree) { + std::discrete_distribution distribution(probabilities.begin(), probabilities.end()); std::vector degrees(numVertices); for (int i = 0; i < numVertices; i++) { - degrees[i] = distribution(generator) + minDegree; + degrees[i] = distribution(rng) + minDegree; } return degrees; } -std::vector GenerateLogNormalDegrees(int numVertices, double mu, - double sigma, int minDegree, - int maxDegree) { +std::vector GenerateLogNormalDegrees(TRNG& rng, int numVertices, double mu, double sigma, int minDegree, int maxDegree) { if (maxDegree == -1) maxDegree = numVertices - 1; std::vector probabilities(maxDegree - minDegree + 1); @@ -450,12 +444,11 @@ std::vector GenerateLogNormalDegrees(int numVertices, double mu, double z = (logX - mu) / sigma; // PDF: (1/(x*σ*√(2π))) * exp(-(ln(x)-μ)²/(2σ²)) - probabilities[k - minDegree] = (1.0 / (x * sigma * std::sqrt(2.0 * M_PI))) * - std::exp(-0.5 * z * z); + probabilities[k - minDegree] = (1.0 / (x * sigma * std::sqrt(2.0 * M_PI))) * std::exp(-0.5 * z * z); } NormalizeProbabilities(probabilities); - return SampleFromPMF(probabilities, numVertices, minDegree); + return SampleFromPMF(rng, probabilities, numVertices, minDegree); } TRelationGraph GenerateRandomChungLuGraph(TRNG &mt, const std::vector& degrees) { diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_generator.h b/ydb/core/kqp/ut/join/kqp_join_topology_generator.h index a19fe9d5f240..d54397c60359 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_generator.h +++ b/ydb/core/kqp/ut/join/kqp_join_topology_generator.h @@ -272,11 +272,13 @@ class TSchemaStats { void NormalizeProbabilities(std::vector& probabilities); std::vector SampleFromPMF( + TRNG& rng, const std::vector& probabilities, int numVertices, int minDegree); std::vector GenerateLogNormalDegrees( - int numVertices, double logMean = 1.0, double logStdDev = 0.5, + TRNG& rng, int numVertices, + double logMean = 1.0, double logStdDev = 0.5, int minDegree = 1, int maxDegree = -1 ); diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp index eeb7ad71e8da..4dafe09465c3 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp @@ -1,5 +1,7 @@ +#include #include #include +#include #include #include #include @@ -316,33 +318,66 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { }; } + struct TBenchState { + ui32 Seed; + ui32 TopologyCounter; + ui32 MCMCCounter; + ui32 KeyCounter; + + std::string toHex() const { + std::stringstream ss; + ss << std::hex << std::setfill('0'); + ss << std::setw(8) << Seed + << std::setw(8) << TopologyCounter + << std::setw(8) << MCMCCounter + << std::setw(8) << KeyCounter; + return ss.str(); + } + + static TBenchState fromHex(const std::string& hex) { + TBenchState state; + ui32* parts[] = {&state.Seed, &state.TopologyCounter, &state.MCMCCounter, &state.KeyCounter}; + for (ui32 i = 0; i < 4; ++ i) { + *parts[i] = std::stoul(hex.substr(i * 8, 8), nullptr, 16); + } + + return state; + } + }; + struct TTestContext { std::unique_ptr Runner; NYdb::NQuery::TQueryClient QueryClient; NYdb::NQuery::TSession Session; + std::optional State; TRNG RNG; + std::string OutputDir; std::map Streams = {}; }; - TTestContext CreateTestContext(TArgs args, uint64_t state = 0, std::string outputDir = "") { - TRNG rng = TRNG::Deserialize(state); + TTestContext CreateTestContext(TArgs args, std::string outputDir = "") { + std::random_device randomDevice; + TRNG rng(randomDevice() % UINT32_MAX); + + std::optional state; + if (args.HasArg("state")) { + state = TBenchState::fromHex(args.GetString("state")); + rng.seed(state->Seed); + } auto numTablesRanged = args.GetArg("N"); - rng.reset(); // ensure this setup is always the same TSchema fullSchema = TSchema::MakeWithEnoughColumns(numTablesRanged.GetLast()); TString stats = TSchemaStats::MakeRandom(rng, fullSchema, 7, 10).ToJSON(); - rng.Restore(state); - auto kikimr = GetCBOTestsYDB(stats, TDuration::Seconds(10)); auto db = kikimr->GetQueryClient(); auto session = db.GetSession().GetValueSync().GetSession(); - return { std::move(kikimr), std::move(db), std::move(session), std::move(rng), outputDir }; + return { std::move(kikimr), std::move(db), std::move(session), state, std::move(rng), outputDir }; } std::string WriteGraph(TTestContext &ctx, ui32 graphID, const TRelationGraph &graph) { @@ -433,7 +468,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { if (topologyName == "mcmc") { return []([[maybe_unused]] TRNG rng, ui32 n, [[maybe_unused]] double mu, [[maybe_unused]] double sigma) { Cout << "================================= METRICS ================================\n"; - auto sampledDegrees = GenerateLogNormalDegrees(n, mu, sigma); + auto sampledDegrees = GenerateLogNormalDegrees(rng, n, mu, sigma); Cout << "sampled degrees: " << JoinSeq(", ", sampledDegrees) << "\n"; auto graphicDegrees = MakeGraphicConnected(sampledDegrees); @@ -447,7 +482,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { if (topologyName == "chung-lu") { return []([[maybe_unused]] TRNG rng, ui32 n, [[maybe_unused]] double mu, [[maybe_unused]] double sigma) { Cout << "================================= METRICS ================================\n"; - auto initialDegrees = GenerateLogNormalDegrees(n, mu, sigma); + auto initialDegrees = GenerateLogNormalDegrees(rng, n, mu, sigma); Cout << "initial degrees: " << JoinSeq(", ", initialDegrees) << "\n"; auto initialGraph = GenerateRandomChungLuGraph(rng, initialDegrees); @@ -461,14 +496,15 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { void RunBenches(TTestContext &ctx, TBenchmarkConfig config, TArgs args) { std::string resultType = args.GetStringOrDefault("result", "SE"); - ui64 generationRepeats = args.GetArgOrDefault("gen", "1").GetValue(); - ui64 repeats = args.GetArgOrDefault("repeats", "1").GetValue(); + ui64 topologyGenerationRepeats = args.GetArgOrDefault("gen-n", "1").GetValue(); + ui64 mcmcRepeats = args.GetArgOrDefault("mcmc-n", "1").GetValue(); + ui64 equiJoinKeysGenerationRepeats = args.GetArgOrDefault("keys-n", "1").GetValue(); std::string topologyName = args.GetStringOrDefault("type", "star"); auto generateTopology = GetTopology(topologyName); std::string headerAggregate = "idx,N,alpha,theta,sigma,mu," + TComputedStatistics::GetCSVHeader(); - std::string header = "idx,seed,graph_name,aggregate_" + headerAggregate; + std::string header = "idx,state,graph_name,aggregate_" + headerAggregate; ui32 idx = 0; ui32 aggregateIdx = 0; @@ -485,45 +521,68 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { << "," << sigma << "," << mu; std::map aggregate; - for (ui64 j = 0; j < generationRepeats; ++ j) { + for (ui64 i = 0; i < topologyGenerationRepeats; ++ i) { Cout << "\n\n\n"; - auto initialGraph = generateTopology(ctx.RNG, n, mu, sigma); - - for (ui64 i = 0; i < repeats; ++ i) { - ui64 seed = ctx.RNG.Serialize(); - Cout << "\n\nReproduce: ya make -r -ttt --test-param TOPOLOGY='"; - Cout << "type=" << topologyName << "; " - << "N=" << n << "; " - << "alpha=" << alpha << "; " - << "theta=" << theta << "; " - << "sigma=" << sigma << "; " - << "mu=" << mu << "; " - << "seed=" << ctx.RNG.Serialize() << "'\n"; + if (ctx.State) { + ctx.RNG.Forward(ctx.State->TopologyCounter); + } + ui32 counterTopology = ctx.RNG.GetCounter(); + auto initialGraph = generateTopology(ctx.RNG, n, mu, sigma); + for (ui64 j = 0; j < mcmcRepeats; ++ j) { TRelationGraph graph = initialGraph; - if (i != 0) { - MCMCRandomize(ctx.RNG, graph); + + if (ctx.State) { + ctx.RNG.Forward(ctx.State->MCMCCounter); } - graph.SetupKeysPitmanYor(ctx.RNG, TPitmanYorConfig{.Alpha = alpha, .Theta = theta}); + ui32 counterMCMC = ctx.RNG.GetCounter(); + if (topologyName == "mcmc") { + MCMCRandomize(ctx.RNG, graph); + } - try { - auto result = BenchmarkShuffleEliminationOnTopology(config, ctx.Session, resultType, graph); - if (!result) { - goto stop; + for (ui64 k = 0; k < equiJoinKeysGenerationRepeats; ++ k) { + if (ctx.State) { + ctx.RNG.Forward(ctx.State->KeyCounter); } - AccumulateAllStats(aggregate, *result); - std::string graphName = WriteGraph(ctx, idx, graph); + ui32 counterKeys = ctx.RNG.GetCounter(); + graph.SetupKeysPitmanYor(ctx.RNG, TPitmanYorConfig{.Alpha = alpha, .Theta = theta}); + + std::string state = TBenchState(ctx.RNG.GetSeed(), counterTopology, counterMCMC, counterKeys).toHex(); + + Cout << "\n\nReproduce: TOPOLOGY='"; + Cout << "type=" << topologyName << "; " + << "N=" << n << "; " + << "alpha=" << alpha << "; " + << "theta=" << theta << "; " + << "sigma=" << sigma << "; " + << "mu=" << mu << "; " + << "state=" << state << "'\n"; + + try { + auto result = BenchmarkShuffleEliminationOnTopology(config, ctx.Session, resultType, graph); + if (!result) { + goto stop; + } + + AccumulateAllStats(aggregate, *result); + std::string graphName = WriteGraph(ctx, idx, graph); + + std::stringstream params; + params << idx ++ << "," << state << "," << graphName << "," << commonParams.str(); + WriteAllStats(ctx, "", header, params.str(), *result); + } catch (std::exception &exc) { + Cout << "Skipped run: " << exc.what() << "\n"; + continue; + } - std::stringstream params; - params << idx ++ << "," << seed << "," << graphName << "," << commonParams.str(); - WriteAllStats(ctx, "", header, params.str(), *result); - } catch (std::exception &exc) { - Cout << "Skipped run: " << exc.what() << "\n"; - continue; + if (ctx.State) { + // We are running in reproducibility mode, stop immediately after case is reproduced + return; + } } } } @@ -550,8 +609,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { auto config = GetBenchmarkConfig(args); DumpBenchmarkConfig(Cout, config); - uint64_t state = args.GetArgOrDefault("seed", "0").GetValue(); - TTestContext ctx = CreateTestContext(args, state, GetTestParam("SAVE_DIR")); + TTestContext ctx = CreateTestContext(args, GetTestParam("SAVE_DIR")); RunBenches(ctx, config, args); } From a224bede3e62be00b61fe023385ea870353c616b Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Wed, 5 Nov 2025 05:35:52 +0000 Subject: [PATCH 34/43] Revert unnecessary changes in kqp_join_order_ut.cpp --- ydb/core/kqp/ut/join/kqp_join_order_ut.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp index 51d70af316a1..2887e80086f7 100644 --- a/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp @@ -1,8 +1,6 @@ #include #include -#include -#include #include #include @@ -11,7 +9,6 @@ #include #include - namespace NKikimr { namespace NKqp { From 1b223d2873b8d13fd0f3d7eea7f60d83d7d5df5f Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Wed, 5 Nov 2025 05:43:08 +0000 Subject: [PATCH 35/43] Add kqp_benches.cpp that was extracted from kqp_benches.h after refactor --- ydb/core/kqp/ut/common/kqp_benches.cpp | 314 +++++++++++++++++++++++++ 1 file changed, 314 insertions(+) create mode 100644 ydb/core/kqp/ut/common/kqp_benches.cpp diff --git a/ydb/core/kqp/ut/common/kqp_benches.cpp b/ydb/core/kqp/ut/common/kqp_benches.cpp new file mode 100644 index 000000000000..b4dc93bb4ad3 --- /dev/null +++ b/ydb/core/kqp/ut/common/kqp_benches.cpp @@ -0,0 +1,314 @@ +#include "kqp_benches.h" +#include + + +namespace NKikimr::NKqp { + + +std::string TComputedStatistics::GetCSVHeader() { + return "median,MAD,Q1,Q3,IQR,mean,stdev,n,min,max"; +} + +void TComputedStatistics::ToCSV(IOutputStream &os) const { + os << Median << "," << MAD + << "," << Q1 << "," << Q3 << "," << IQR + << "," << Mean << "," << Stdev + << "," << N << "," << Min << "," << Max; +} + + +double CalculatePercentile(std::vector& data, double percentile) { + if (data.empty()) { + throw std::invalid_argument("At least one element needed to get percentiles"); + } + + double position = percentile * (data.size() - 1); + + // Get range of positions that are considered to be at this percentile + ui64 lower = std::floor(position); + ui64 upper = std::ceil(position); + double weight = position - lower; + + std::nth_element(data.begin(), data.begin() + lower, data.end()); + double lowerValue = data[lower]; + + // If position is precise, just get that element: + if (lower == upper) { + return lowerValue; + } + + // If it's a range, then interpolate linearly + // e.g. median of even number of elements is average of the central two + std::nth_element(data.begin(), data.begin() + upper, data.end()); + double upperValue = data[upper]; + + return lowerValue + weight * (upperValue - lowerValue); +} + +double CalculateMedian(std::vector& data) { + return CalculatePercentile(data, 0.5); +} + +double CalculateMAD(const std::vector& data, double median, std::vector &storage) { + storage.clear(); + storage.reserve(data.size()); + + for (double value : data) { + storage.push_back(abs(value - median)); + } + + return CalculatePercentile(storage, 0.5); +} + + +double CalculateMean(const std::vector &data) { + return std::accumulate(data.begin(), data.end(), 0.0) + / static_cast(data.size()); +} + + +double CalculateSampleStdDev(const std::vector &data, double mean) { + if (data.size() == 0) { + return 0; + } + + double deviationSquaresSum = 0; + for (double value : data) { + double deviation = abs(value - mean); + deviationSquaresSum += deviation * deviation; + }; + + + return std::sqrt(deviationSquaresSum / static_cast(data.size() - 1)); +} + + +void TRunningStatistics::AddValue(double num) { + Samples_.push_back(num); + Total_ += num; + + if (!Min_ || !Max_) { + Min_ = num; + Max_ = num; + } + + Min_ = Min(*Min_, num); + Max_ = Max(*Max_, num); + + if (MaxHeap_.empty() || num <= MaxHeap_.top()) { + MaxHeap_.push(num); + } else { + MinHeap_.push(num); + } + + if (MaxHeap_.size() > MinHeap_.size() + 1) { + MinHeap_.push(MaxHeap_.top()); + MaxHeap_.pop(); + } else if (MinHeap_.size() > MaxHeap_.size()) { + MaxHeap_.push(MinHeap_.top()); + MinHeap_.pop(); + } +} + +double TRunningStatistics::GetMedian() const { + assert(!MaxHeap_.empty() || !MinHeap_.empty()); + + if (MaxHeap_.size() > MinHeap_.size()) { + return MaxHeap_.top(); + } else { + return (MaxHeap_.top() + MinHeap_.top()) / 2.0; + } +} + + +double TRunningStatistics::CalculateMAD() const { + return ::NKikimr::NKqp::CalculateMAD(Samples_, GetMedian(), MADStorage_); +} + +TComputedStatistics TStatistics::ComputeStatistics() const { + double q1 = CalculatePercentile(Samples_, 0.25); + double q3 = CalculatePercentile(Samples_, 0.75); + double iqr = q3 - q1; + + double median = CalculateMedian(Samples_); + + // ComputeStatistics should be called once per statistics combiner at most. + // Otherwise we would hold on to storage instead of reallocating it each time: + std::vector storage; + double mad = ::NKikimr::NKqp::CalculateMAD(Samples_, median, storage); + + double mean = CalculateMean(Samples_); + double stdev = CalculateSampleStdDev(Samples_, mean); + + + return { + .N = Samples_.size(), + .Min = Min_, + .Max = Max_, + .Median = median, + .MAD = mad, + .Q1 = q1, + .Q3 = q3, + .IQR = iqr, + .Mean = mean, + .Stdev = stdev, + }; +} + +TStatistics TStatistics::operator-(const TStatistics &statsRHS) const { + auto op = [](double lhs, double rhs) { + return lhs - rhs; + }; + + return TStatistics { + Combine(statsRHS, op), + Min_ - statsRHS.Max_, + Max_ - statsRHS.Min_ + }; +} + +TStatistics TStatistics::operator+(const TStatistics &statsRHS) const { + auto op = [](double lhs, double rhs) { + return lhs - rhs; + }; + + return TStatistics { + Combine(statsRHS, op), + Min_ + statsRHS.Min_, + Max_ + statsRHS.Max_ + }; +} + +TStatistics TStatistics::operator*(const TStatistics &statsRHS) const { + auto op = [](double lhs, double rhs) { + return lhs - rhs; + }; + + double candidates[] = { + Min_ * statsRHS.Min_, + Min_ * statsRHS.Max_, + Max_ * statsRHS.Min_, + Max_ * statsRHS.Max_, + }; + + return TStatistics { + Combine(statsRHS, op), + *std::min_element(candidates, candidates + Y_ARRAY_SIZE(candidates)), + *std::max_element(candidates, candidates + Y_ARRAY_SIZE(candidates)) + }; +} + +TStatistics TStatistics::operator/(const TStatistics &statsRHS) const { + auto op = [](double lhs, double rhs) { + return lhs / rhs; + }; + + double candidates[] = { + Min_ / statsRHS.Min_, + Min_ / statsRHS.Max_, + Max_ / statsRHS.Min_, + Max_ / statsRHS.Max_, + }; + + return TStatistics { + Combine(statsRHS, op), + *std::min_element(candidates, candidates + Y_ARRAY_SIZE(candidates)), + *std::max_element(candidates, candidates + Y_ARRAY_SIZE(candidates)) + }; +} + + +static std::pair SelectUnit(ui64 nanoseconds) { + if (nanoseconds >= 1'000'000'000) { + return {"s", 1e9}; + } else if (nanoseconds >= 1'000'000) { + return {"ms", 1e6}; + } else if (nanoseconds >= 1'000) { + return {"μs", 1e3}; + } else { + return {"ns", 1.0}; + } +} + +static int GetDecimalPlaces(double scaledUncertainty) { + if (scaledUncertainty < 0.01) return 2; + + double log_val = std::log10(scaledUncertainty); + int magnitude = static_cast(std::floor(log_val)); + + if (scaledUncertainty >= 100.0) { + return 0; + } else if (scaledUncertainty >= 10.0) { + return 1; + } else if (scaledUncertainty >= 1.0) { + return 1; + } else { + return std::max(0, -magnitude + 1); + } +} + +static int GetDecimalPlacesForValue(double scaledValue) { + if (scaledValue < 0.01) return 4; + + double absValue = std::abs(scaledValue); + + if (absValue >= 1000.0) { + return 0; + } else if (absValue >= 100.0) { + return 1; + } else if (absValue >= 10.0) { + return 2; + } else if (absValue >= 1.0) { + return 2; + } else if (absValue >= 0.1) { + return 3; + } else { + return 4; + } +} + +std::string TimeFormatter::Format(ui64 valueNs, ui64 uncertaintyNs) { + auto [unit, scale] = SelectUnit(valueNs); + + double scaledValue = valueNs / scale; + double scaledUncertainty = uncertaintyNs / scale; + + int decimalPlaces = GetDecimalPlaces(scaledUncertainty); + + std::ostringstream oss; + oss << std::fixed << std::setprecision(decimalPlaces); + oss << scaledValue << " " << unit << " ± " << scaledUncertainty << " " << unit; + + return oss.str(); +} + +std::string TimeFormatter::Format(ui64 valueNs) { + auto [unit, scale] = SelectUnit(valueNs); + double scaledValue = valueNs / scale; + + int decimalPlaces = GetDecimalPlacesForValue(scaledValue); + + std::ostringstream oss; + oss << std::fixed << std::setprecision(decimalPlaces); + oss << scaledValue << " " << unit; + + return oss.str(); +} + +void DumpTimeStatistics(const TComputedStatistics &stats, IOutputStream &OS) { + OS << "Median = " << TimeFormatter::Format(stats.Median, stats.MAD) << " (MAD)\n"; + + OS << "Q1 = " << TimeFormatter::Format(stats.Q1) + << ", Q3 = " << TimeFormatter::Format(stats.Q3) + << ", IQR = " < Date: Thu, 6 Nov 2025 14:22:47 +0000 Subject: [PATCH 36/43] Fix bug in TStatistics::Merge - incorrect min of a distribution --- ydb/core/kqp/ut/common/kqp_benches.cpp | 6 ++++++ ydb/core/kqp/ut/common/kqp_benches.h | 6 +----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/ydb/core/kqp/ut/common/kqp_benches.cpp b/ydb/core/kqp/ut/common/kqp_benches.cpp index b4dc93bb4ad3..8a0d53d66c09 100644 --- a/ydb/core/kqp/ut/common/kqp_benches.cpp +++ b/ydb/core/kqp/ut/common/kqp_benches.cpp @@ -155,6 +155,12 @@ TComputedStatistics TStatistics::ComputeStatistics() const { }; } +void TStatistics::Merge(const TStatistics& other) { + Samples_.insert(Samples_.end(), other.Samples_.begin(), other.Samples_.end()); + Min_ = std::min(Min_, other.Min_); + Max_ = std::max(Max_, other.Max_); +} + TStatistics TStatistics::operator-(const TStatistics &statsRHS) const { auto op = [](double lhs, double rhs) { return lhs - rhs; diff --git a/ydb/core/kqp/ut/common/kqp_benches.h b/ydb/core/kqp/ut/common/kqp_benches.h index 6245285f5087..f782f065abca 100644 --- a/ydb/core/kqp/ut/common/kqp_benches.h +++ b/ydb/core/kqp/ut/common/kqp_benches.h @@ -80,11 +80,7 @@ class TStatistics { return Max_; } - void Merge(TStatistics other) { - Samples_.insert(Samples_.end(), other.Samples_.begin(), other.Samples_.end()); - Min_ = std::min(Min_, other.Min_); - Max_ = std::min(Max_, other.Max_); - } + void Merge(const TStatistics& other); // Operations on two random values, very expensive. // Each operation produces cartesian product of all possibilities if From 433a4f49493473e818a658aa58288e3d22e32e97 Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Thu, 6 Nov 2025 14:26:04 +0000 Subject: [PATCH 37/43] Don't reject samples where Shuffle Elimination was faster --- ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp index 4dafe09465c3..21e8b58b999f 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp @@ -170,7 +170,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { auto& adjustedWithoutTime = results.at("CBO-0"); if (adjustedWithTime.ComputeStatistics().Median > adjustedWithoutTime.ComputeStatistics().Median) { - auto adjustedRatio = (adjustedWithTime / adjustedWithoutTime).Filter([](double value) { return value >= 1.0; }); + auto adjustedRatio = adjustedWithTime / adjustedWithoutTime; results.emplace("SE-0-div-CBO-0", adjustedRatio); } } From 2a4bbd7719330d73ed7a04d800d5e4fd66c50fb9 Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Thu, 6 Nov 2025 14:26:52 +0000 Subject: [PATCH 38/43] Prevent NANs in speed ratio calculation --- ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp index 21e8b58b999f..6845162dafab 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp @@ -130,7 +130,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { withoutShuffleElimination = BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/false, /*optLevel=*/2)); if (resultType.contains("0")) { - results.emplace("CBO-0", (*withoutShuffleElimination - *withoutCBO).Filter([](double value) { return value >= 0; })); + results.emplace("CBO-0", (*withoutShuffleElimination - *withoutCBO).Filter([](double value) { return value > 0; })); } if (!withoutShuffleElimination) { @@ -149,7 +149,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { withShuffleElimination = BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/true, /*optLevel=*/2)); if (resultType.contains("0")) { - results.emplace("SE-0", (*withShuffleElimination - *withoutCBO).Filter([](double value) { return value >= 0; })); + results.emplace("SE-0", (*withShuffleElimination - *withoutCBO).Filter([](double value) { return value > 0; })); } if (!withShuffleElimination) { From 0ac5488973e65a238acfd61bddeb70023bdd0a81 Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Thu, 6 Nov 2025 21:29:06 +0000 Subject: [PATCH 39/43] Fix styling + minor fixes --- ydb/core/kqp/opt/logical/kqp_opt_log.cpp | 2 +- ydb/core/kqp/ut/common/kqp_arg_parser.h | 37 ++- ydb/core/kqp/ut/common/kqp_benches.cpp | 71 ++--- ydb/core/kqp/ut/common/kqp_benches.h | 57 ++-- ydb/core/kqp/ut/common/kqp_serializable_rng.h | 30 +- .../ut/join/kqp_join_topology_generator.cpp | 297 +++++++++++------- .../kqp/ut/join/kqp_join_topology_generator.h | 186 ++++------- ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp | 201 +++++------- .../yql/dq/opt/dq_opt_join_cbo_factory.cpp | 2 +- .../yql/dq/opt/dq_opt_join_cost_based.cpp | 6 +- .../yql/dq/opt/dq_opt_join_cost_based.h | 2 +- ydb/library/yql/dq/opt/ut/dq_cbo_ut.cpp | 12 +- .../yql/dq/opt/ut/dq_opt_hypergraph_ut.cpp | 2 +- yql/essentials/core/cbo/cbo_optimizer_new.h | 4 +- yql/essentials/core/cbo/simple/cbo_simple.cpp | 2 +- 15 files changed, 439 insertions(+), 472 deletions(-) diff --git a/ydb/core/kqp/opt/logical/kqp_opt_log.cpp b/ydb/core/kqp/opt/logical/kqp_opt_log.cpp index 0b76357c8591..34e618228301 100644 --- a/ydb/core/kqp/opt/logical/kqp_opt_log.cpp +++ b/ydb/core/kqp/opt/logical/kqp_opt_log.cpp @@ -175,7 +175,7 @@ class TKqpLogicalOptTransformer : public TOptimizeTransformerBase { } TMaybeNode OptimizeEquiJoinWithCosts(TExprBase node, TExprContext& ctx) { - TOptimizerSettings settings { + TCBOSettings settings{ .MaxDPhypDPTableSize = Config->MaxDPHypDPTableSize.Get().GetOrElse(TDqSettings::TDefault::MaxDPHypDPTableSize), .ShuffleEliminationJoinNumCutoff = Config->ShuffleEliminationJoinNumCutoff.Get().GetOrElse(TDqSettings::TDefault::ShuffleEliminationJoinNumCutoff) }; diff --git a/ydb/core/kqp/ut/common/kqp_arg_parser.h b/ydb/core/kqp/ut/common/kqp_arg_parser.h index febc536e9467..ed1728983e52 100644 --- a/ydb/core/kqp/ut/common/kqp_arg_parser.h +++ b/ydb/core/kqp/ut/common/kqp_arg_parser.h @@ -10,10 +10,8 @@ #include #include - namespace NKikimr::NKqp { - class TArgs { public: template @@ -144,24 +142,23 @@ class TArgs { return Values_.contains(key); } - private: std::map Values_; private: - static void LTrim(std::string &input) { + static void LTrim(std::string& input) { input.erase(input.begin(), std::find_if(input.begin(), input.end(), [](unsigned char ch) { return !std::isspace(ch); })); } - static void RTrim(std::string &input) { + static void RTrim(std::string& input) { input.erase(std::find_if(input.rbegin(), input.rend(), [](unsigned char ch) { return !std::isspace(ch); }).base(), input.end()); } - static void Trim(std::string &input) { + static void Trim(std::string& input) { LTrim(input); RTrim(input); } @@ -243,17 +240,27 @@ class TArgs { double value = std::stod(match[1]); std::string unit = match[2]; - if (unit == "ns") return std::chrono::nanoseconds(static_cast(value)); - if (unit == "us") return std::chrono::microseconds(static_cast(value)); - if (unit == "ms") return std::chrono::milliseconds(static_cast(value)); - if (unit == "s") return std::chrono::seconds(static_cast(value)); - if (unit == "m") return std::chrono::minutes(static_cast(value)); - if (unit == "h") return std::chrono::hours(static_cast(value)); + if (unit == "ns") { + return std::chrono::nanoseconds(static_cast(value)); + } + if (unit == "us") { + return std::chrono::microseconds(static_cast(value)); + } + if (unit == "ms") { + return std::chrono::milliseconds(static_cast(value)); + } + if (unit == "s") { + return std::chrono::seconds(static_cast(value)); + } + if (unit == "m") { + return std::chrono::minutes(static_cast(value)); + } + if (unit == "h") { + return std::chrono::hours(static_cast(value)); + } throw std::invalid_argument("Unknown unit"); } - }; - -} +} // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/ut/common/kqp_benches.cpp b/ydb/core/kqp/ut/common/kqp_benches.cpp index 8a0d53d66c09..a2770401ee40 100644 --- a/ydb/core/kqp/ut/common/kqp_benches.cpp +++ b/ydb/core/kqp/ut/common/kqp_benches.cpp @@ -1,22 +1,21 @@ #include "kqp_benches.h" -#include +#include +#include namespace NKikimr::NKqp { - std::string TComputedStatistics::GetCSVHeader() { return "median,MAD,Q1,Q3,IQR,mean,stdev,n,min,max"; } -void TComputedStatistics::ToCSV(IOutputStream &os) const { +void TComputedStatistics::ToCSV(IOutputStream& os) const { os << Median << "," << MAD << "," << Q1 << "," << Q3 << "," << IQR << "," << Mean << "," << Stdev << "," << N << "," << Min << "," << Max; } - double CalculatePercentile(std::vector& data, double percentile) { if (data.empty()) { throw std::invalid_argument("At least one element needed to get percentiles"); @@ -49,7 +48,7 @@ double CalculateMedian(std::vector& data) { return CalculatePercentile(data, 0.5); } -double CalculateMAD(const std::vector& data, double median, std::vector &storage) { +double CalculateMAD(const std::vector& data, double median, std::vector& storage) { storage.clear(); storage.reserve(data.size()); @@ -60,14 +59,11 @@ double CalculateMAD(const std::vector& data, double median, std::vector< return CalculatePercentile(storage, 0.5); } - -double CalculateMean(const std::vector &data) { - return std::accumulate(data.begin(), data.end(), 0.0) - / static_cast(data.size()); +double CalculateMean(const std::vector& data) { + return std::accumulate(data.begin(), data.end(), 0.0) / static_cast(data.size()); } - -double CalculateSampleStdDev(const std::vector &data, double mean) { +double CalculateSampleStdDev(const std::vector& data, double mean) { if (data.size() == 0) { return 0; } @@ -78,11 +74,9 @@ double CalculateSampleStdDev(const std::vector &data, double mean) { deviationSquaresSum += deviation * deviation; }; - return std::sqrt(deviationSquaresSum / static_cast(data.size() - 1)); } - void TRunningStatistics::AddValue(double num) { Samples_.push_back(num); Total_ += num; @@ -111,7 +105,7 @@ void TRunningStatistics::AddValue(double num) { } double TRunningStatistics::GetMedian() const { - assert(!MaxHeap_.empty() || !MinHeap_.empty()); + Y_ASSERT(!MaxHeap_.empty() || !MinHeap_.empty()); if (MaxHeap_.size() > MinHeap_.size()) { return MaxHeap_.top(); @@ -120,7 +114,6 @@ double TRunningStatistics::GetMedian() const { } } - double TRunningStatistics::CalculateMAD() const { return ::NKikimr::NKqp::CalculateMAD(Samples_, GetMedian(), MADStorage_); } @@ -140,7 +133,6 @@ TComputedStatistics TStatistics::ComputeStatistics() const { double mean = CalculateMean(Samples_); double stdev = CalculateSampleStdDev(Samples_, mean); - return { .N = Samples_.size(), .Min = Min_, @@ -161,31 +153,31 @@ void TStatistics::Merge(const TStatistics& other) { Max_ = std::max(Max_, other.Max_); } -TStatistics TStatistics::operator-(const TStatistics &statsRHS) const { +TStatistics TStatistics::operator-(const TStatistics& statsRHS) const { auto op = [](double lhs, double rhs) { return lhs - rhs; }; - return TStatistics { + return TStatistics{ Combine(statsRHS, op), Min_ - statsRHS.Max_, Max_ - statsRHS.Min_ }; } -TStatistics TStatistics::operator+(const TStatistics &statsRHS) const { +TStatistics TStatistics::operator+(const TStatistics& statsRHS) const { auto op = [](double lhs, double rhs) { return lhs - rhs; }; - return TStatistics { + return TStatistics{ Combine(statsRHS, op), Min_ + statsRHS.Min_, Max_ + statsRHS.Max_ }; } -TStatistics TStatistics::operator*(const TStatistics &statsRHS) const { +TStatistics TStatistics::operator*(const TStatistics& statsRHS) const { auto op = [](double lhs, double rhs) { return lhs - rhs; }; @@ -197,14 +189,14 @@ TStatistics TStatistics::operator*(const TStatistics &statsRHS) const { Max_ * statsRHS.Max_, }; - return TStatistics { + return TStatistics{ Combine(statsRHS, op), *std::min_element(candidates, candidates + Y_ARRAY_SIZE(candidates)), *std::max_element(candidates, candidates + Y_ARRAY_SIZE(candidates)) }; } -TStatistics TStatistics::operator/(const TStatistics &statsRHS) const { +TStatistics TStatistics::operator/(const TStatistics& statsRHS) const { auto op = [](double lhs, double rhs) { return lhs / rhs; }; @@ -216,14 +208,13 @@ TStatistics TStatistics::operator/(const TStatistics &statsRHS) const { Max_ / statsRHS.Max_, }; - return TStatistics { + return TStatistics{ Combine(statsRHS, op), *std::min_element(candidates, candidates + Y_ARRAY_SIZE(candidates)), *std::max_element(candidates, candidates + Y_ARRAY_SIZE(candidates)) }; } - static std::pair SelectUnit(ui64 nanoseconds) { if (nanoseconds >= 1'000'000'000) { return {"s", 1e9}; @@ -237,10 +228,12 @@ static std::pair SelectUnit(ui64 nanoseconds) { } static int GetDecimalPlaces(double scaledUncertainty) { - if (scaledUncertainty < 0.01) return 2; + if (scaledUncertainty < 0.01) { + return 2; + } - double log_val = std::log10(scaledUncertainty); - int magnitude = static_cast(std::floor(log_val)); + double log = std::log10(scaledUncertainty); + int magnitude = static_cast(std::floor(log)); if (scaledUncertainty >= 100.0) { return 0; @@ -254,7 +247,9 @@ static int GetDecimalPlaces(double scaledUncertainty) { } static int GetDecimalPlacesForValue(double scaledValue) { - if (scaledValue < 0.01) return 4; + if (scaledValue < 0.01) { + return 4; + } double absValue = std::abs(scaledValue); @@ -301,20 +296,20 @@ std::string TimeFormatter::Format(ui64 valueNs) { return oss.str(); } -void DumpTimeStatistics(const TComputedStatistics &stats, IOutputStream &OS) { - OS << "Median = " << TimeFormatter::Format(stats.Median, stats.MAD) << " (MAD)\n"; +void DumpTimeStatistics(const TComputedStatistics& stats, IOutputStream& os) { + os << "Median = " << TimeFormatter::Format(stats.Median, stats.MAD) << " (MAD)\n"; - OS << "Q1 = " << TimeFormatter::Format(stats.Q1) + os << "Q1 = " << TimeFormatter::Format(stats.Q1) << ", Q3 = " << TimeFormatter::Format(stats.Q3) - << ", IQR = " < -#include -#include #include #include #include #include #include -#include -#include #include -#include -#include +#include #include - +#include +#include namespace NKikimr::NKqp { - template -ui64 MeasureTimeNanos(Lambda &&lambda) { +ui64 MeasureTimeNanos(Lambda&& lambda) { namespace Nsc = std::chrono; using TClock = Nsc::high_resolution_clock; @@ -30,7 +24,6 @@ ui64 MeasureTimeNanos(Lambda &&lambda) { return Nsc::duration_cast(TClock::now() - start).count(); } - struct TComputedStatistics { ui64 N; double Min, Max; @@ -46,17 +39,15 @@ struct TComputedStatistics { double Stdev; static std::string GetCSVHeader(); - void ToCSV(IOutputStream &os) const; + void ToCSV(IOutputStream& os) const; }; - double CalculatePercentile(const std::vector& data, double percentile); double CalculateMedian(std::vector& data); double CalculateMAD(const std::vector& data, double median, std::vector& storage); double CalculateMean(const std::vector& data); double CalculateSampleStdDev(const std::vector& data, double mean); - class TStatistics { public: TStatistics(std::vector samples, double min, double max) @@ -85,14 +76,14 @@ class TStatistics { // Operations on two random values, very expensive. // Each operation produces cartesian product of all possibilities if // it's small enough or fallbacks to Monte Carlo methods if not. - TStatistics operator-(const TStatistics &rhs) const; - TStatistics operator+(const TStatistics &rhs) const; - TStatistics operator*(const TStatistics &rhs) const; - TStatistics operator/(const TStatistics &rhs) const; + TStatistics operator-(const TStatistics& rhs) const; + TStatistics operator+(const TStatistics& rhs) const; + TStatistics operator*(const TStatistics& rhs) const; + TStatistics operator/(const TStatistics& rhs) const; // Only gives precisely correct min & max if you map with monotonic function template - TStatistics Map(TMapLambda map) { + TStatistics Map(TMapLambda&& map) { std::vector samples; double min = std::numeric_limits::max(); double max = std::numeric_limits::min(); @@ -105,7 +96,7 @@ class TStatistics { max = std::max(newValue, max); } - double candidates[] = { min, max, map(Min_), map(Max_) }; + double candidates[] = {min, max, map(Min_), map(Max_)}; return { std::move(samples), *std::min_element(candidates, candidates + Y_ARRAY_SIZE(candidates)), @@ -114,7 +105,7 @@ class TStatistics { } template - TStatistics Filter(TFilterLambda &&filter) const { + TStatistics Filter(TFilterLambda&& filter) const { std::vector samples; // That's an upper bound since this function can only remove elements. @@ -141,7 +132,6 @@ class TStatistics { return {std::move(samples), globalMin, globalMax}; } - private: mutable std::vector Samples_; double Min_; @@ -150,8 +140,7 @@ class TStatistics { // This lets us operate on two random values, it computes distribution of values after arbitrary operation template std::vector - Combine(const TStatistics& rhs, TCombiner&& combine, ui64 maxSamples = 1e8) const { - + Combine(const TStatistics& rhs, TCombiner&& combine, ui64 maxSamples = 1e8) const { std::vector combined; ui64 cartesianProductSize = Samples_.size() * rhs.Samples_.size(); @@ -184,10 +173,8 @@ class TStatistics { return combined; } - }; - class TRunningStatistics { public: TRunningStatistics() @@ -215,17 +202,17 @@ class TRunningStatistics { } double GetMin() const { - assert(Min_); + Y_ASSERT(Min_); return *Min_; } double GetMax() const { - assert(Max_); + Y_ASSERT(Max_); return *Max_; } TStatistics GetStatistics() { - assert(GetN() > 0); + Y_ASSERT(GetN() > 0); return {Samples_, *Min_, *Max_}; } @@ -245,16 +232,13 @@ class TRunningStatistics { std::optional Max_; }; - - class TimeFormatter { public: static std::string Format(uint64_t valueNs, ui64 uncertaintyNs); static std::string Format(uint64_t valueNs); }; -void DumpTimeStatistics(const TComputedStatistics &stats, IOutputStream &OS); - +void DumpTimeStatistics(const TComputedStatistics& stats, IOutputStream& os); struct TRepeatedTestConfig { ui64 MinRepeats; @@ -263,8 +247,8 @@ struct TRepeatedTestConfig { }; template -std::optional RepeatedTest(TRepeatedTestConfig config, ui64 singleRunTimeout, double thresholdMAD, TLambda &&lambda) { - assert(config.MinRepeats >= 1); +std::optional RepeatedTest(TRepeatedTestConfig config, ui64 singleRunTimeout, double thresholdMAD, TLambda&& lambda) { + Y_ASSERT(config.MinRepeats >= 1); TRunningStatistics stats; do { @@ -288,7 +272,6 @@ std::optional RepeatedTest(TRepeatedTestConfig config, ui64 singleR return std::move(stats).GetStatistics(); } - struct TBenchmarkConfig { TRepeatedTestConfig Warmup; TRepeatedTestConfig Bench; @@ -297,7 +280,7 @@ struct TBenchmarkConfig { }; template -std::optional Benchmark(TBenchmarkConfig config, Lambda &&lambda) { +std::optional Benchmark(TBenchmarkConfig config, Lambda&& lambda) { if (config.Warmup.MaxRepeats != 0) { auto warmupStats = RepeatedTest(config.Warmup, config.SingleRunTimeout, config.MADThreshold, lambda); if (!warmupStats) { diff --git a/ydb/core/kqp/ut/common/kqp_serializable_rng.h b/ydb/core/kqp/ut/common/kqp_serializable_rng.h index 466a49909b8b..8c21ad745859 100644 --- a/ydb/core/kqp/ut/common/kqp_serializable_rng.h +++ b/ydb/core/kqp/ut/common/kqp_serializable_rng.h @@ -4,10 +4,8 @@ #include #include - namespace NKikimr::NKqp { - // wrapper around std::mt19937 that tracks usage and simplifies serialization class TSerializableMT19937 { public: // compatibility with std::mt19937 @@ -19,8 +17,8 @@ class TSerializableMT19937 { : TSerializableMT19937(default_seed) { } - - TSerializableMT19937(uint32_t seed) + + TSerializableMT19937(uint32_t seed) : Engine_(seed) , Seed_(seed) , Counter_(0) @@ -34,7 +32,7 @@ class TSerializableMT19937 { void Restore(uint64_t state) { Seed_ = static_cast(state >> 32); Counter_ = static_cast(state & 0xFFFFFFFF); - + Engine_.seed(Seed_); Engine_.discard(Counter_); } @@ -52,7 +50,7 @@ class TSerializableMT19937 { uint32_t GetSeed() const { return Seed_; } - + static TSerializableMT19937 Deserialize(uint64_t key) { TSerializableMT19937 mt; mt.Restore(key); @@ -61,8 +59,12 @@ class TSerializableMT19937 { } public: // compatibility with std::mt19937 - static constexpr auto min() { return std::mt19937::min(); } - static constexpr auto max() { return std::mt19937::max(); } + static constexpr auto min() { + return std::mt19937::min(); + } + static constexpr auto max() { + return std::mt19937::max(); + } auto operator()() { assert(Counter_ != UINT32_MAX); @@ -76,23 +78,23 @@ class TSerializableMT19937 { Engine_.seed(seed); } - + void discard(uint64_t n) { assert(n <= static_cast(UINT32_MAX - Counter_)); Counter_ += static_cast(n); Engine_.discard(n); } - + void reset() { Counter_ = 0; Engine_.seed(Seed_); } - + bool operator==(const TSerializableMT19937& other) const { return Seed_ == other.Seed_ && Counter_ == other.Counter_; } - + bool operator!=(const TSerializableMT19937& other) const { return !(*this == other); } @@ -101,8 +103,6 @@ class TSerializableMT19937 { std::mt19937 Engine_; uint32_t Seed_; uint32_t Counter_; - }; -} - +} // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp index ec4c96d41f4e..6ec9eb6d9d0b 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_generator.cpp @@ -2,12 +2,10 @@ #include #include -#include #include #include #include - namespace NKikimr::NKqp { static std::string getTableName(unsigned tableID) { @@ -27,16 +25,16 @@ static std::string getTablePath(unsigned tableID) { return "/Root/" + getTableName(tableID); } -static double getRandomNormalizedDouble(TRNG &mt) { +static double getRandomNormalizedDouble(TRNG& rng) { std::uniform_real_distribution<> distribution(0.0, 1.0); - return distribution(mt); + return distribution(rng); } -static std::vector GetPitmanYor(TRNG &mt, ui32 sum, TPitmanYorConfig config) { +static std::vector GetPitmanYor(TRNG& rng, ui32 sum, TPitmanYorConfig config) { std::vector keyCounts{/*initial key=*/1}; for (ui32 column = 1; column < sum; ++ column) { - double random = getRandomNormalizedDouble(mt); + double random = getRandomNormalizedDouble(rng); double cumulative = 0.0; int chosenTableIndex = -1; @@ -60,20 +58,20 @@ static std::vector GetPitmanYor(TRNG &mt, ui32 sum, TPitmanYorConfig confi return keyCounts; } -void TRelationGraph::SetupKeysPitmanYor(TRNG &mt, TPitmanYorConfig config) { - std::vector, /*last index*/ui32>> distributions(GetN()); +void TRelationGraph::SetupKeysPitmanYor(TRNG& rng, TPitmanYorConfig config) { + std::vector, /*last index*/ ui32>> distributions(GetN()); for (ui32 i = 0; i < GetN(); ++ i) { - auto distribution = GetPitmanYor(mt, AdjacencyList_[i].size(), config); + auto distribution = GetPitmanYor(rng, AdjacencyList_[i].size(), config); Schema_[i] = TTable{static_cast(distribution.size())}; distributions[i] = {std::move(distribution), /*initial index=*/0}; } auto GetKey = [&](ui32 node) { - auto &[nodeDistribution, lastIndex] = distributions[node]; + auto& [nodeDistribution, lastIndex] = distributions[node]; ui32 key = lastIndex; - assert(lastIndex < nodeDistribution.size()); - assert(nodeDistribution[lastIndex] != 0); + Y_ASSERT(lastIndex < nodeDistribution.size()); + Y_ASSERT(nodeDistribution[lastIndex] != 0); if (-- nodeDistribution[lastIndex] == 0) { ++ lastIndex; } @@ -112,10 +110,8 @@ void TRelationGraph::SetupKeysPitmanYor(TRNG &mt, TPitmanYorConfig config) { backwardEdge->ColumnRHS = ColumnLHS; } } - } - TSchema TSchema::MakeWithEnoughColumns(unsigned numNodes) { return TSchema{std::vector(numNodes, TTable{numNodes})}; } @@ -155,48 +151,57 @@ void TSchema::Rename(std::vector oldToNew) { Tables_ = newTables; } +void TRelationGraph::Connect(unsigned lhs, unsigned rhs) { + AdjacencyList_[lhs].push_back({/*Target=*/rhs, 0, 0}); + AdjacencyList_[rhs].push_back({/*Target=*/lhs, 0, 0}); +} -TRelationGraph TRelationGraph::FromPrufer(const std::vector& prufer) { - unsigned n = prufer.size() + 2; +void TRelationGraph::Disconnect(unsigned u, unsigned v) { + auto& adjacencyU = AdjacencyList_[u]; + auto& adjacencyV = AdjacencyList_[v]; + adjacencyU.erase(std::remove_if(adjacencyU.begin(), adjacencyU.end(), [v](TEdge edge) { return edge.Target == v; }), adjacencyU.end()); + adjacencyV.erase(std::remove_if(adjacencyV.begin(), adjacencyV.end(), [u](TEdge edge) { return edge.Target == u; }), adjacencyV.end()); +} - std::vector degree(n, 1); - for (unsigned i : prufer) { - ++ degree[i]; +bool TRelationGraph::HasEdge(unsigned u, unsigned v) const { + for (ui32 i = 0; i < AdjacencyList_[u].size(); ++i) { + if (AdjacencyList_[u][i].Target == v) { + return true; + } } - TRelationGraph graph(n); + return false; +} - for (unsigned u : prufer) { - for (unsigned v = 0; v < n; ++ v) { - if (degree[v] == 1) { - graph.Connect(u, v); +std::vector TRelationGraph::FindComponents() const { + std::vector component(GetN(), -1); + int numComponents = 0; - -- degree[v]; - -- degree[u]; - break; - } + for (unsigned start = 0; start < GetN(); ++ start) { + if (component[start] != -1) { + continue; } - } - int u = -1; - unsigned v = 0; - for (; v < n; ++ v) { - if (degree[v] == 1) { - if (u != -1) { - graph.Connect(u, v); - break; + std::queue queue; + queue.push(start); + component[start] = numComponents; + + while (!queue.empty()) { + unsigned u = queue.front(); + queue.pop(); + for (TEdge edge : AdjacencyList_[u]) { + unsigned v = edge.Target; + if (component[v] == -1) { + component[v] = numComponents; + queue.push(v); + } } - - u = v; } - } - return graph; -} + ++ numComponents; + } -void TRelationGraph::Connect(unsigned lhs, unsigned rhs) { - AdjacencyList_[lhs].push_back({/*Target=*/rhs, 0, 0}); - AdjacencyList_[rhs].push_back({/*Target=*/lhs, 0, 0}); + return component; } std::string TRelationGraph::MakeQuery() const { @@ -217,7 +222,6 @@ std::string TRelationGraph::MakeQuery() const { currentJoin += " " + getRelationName(i, AdjacencyList_[i][j].ColumnLHS) + " = " + getRelationName(AdjacencyList_[i][j].Target, AdjacencyList_[i][j].ColumnRHS); - }; for (unsigned j = 0; j < AdjacencyList_[i].size(); ++ j) { @@ -240,34 +244,34 @@ std::string TRelationGraph::MakeQuery() const { // remove extra '\n' from last join if (!joinClause.empty()) { - assert(joinClause.back() == '\n'); + Y_ASSERT(joinClause.back() == '\n'); joinClause.pop_back(); } return std::move(fromClause) + std::move(joinClause) + ";\n"; } -void TRelationGraph::DumpGraph(IOutputStream &OS) const { - OS << "graph {\n"; +void TRelationGraph::DumpGraph(IOutputStream& os) const { + os << "graph {\n"; for (unsigned i = 0; i < AdjacencyList_.size(); ++ i) { for (unsigned j = 0; j < AdjacencyList_[i].size(); ++ j) { - const TEdge &edge = AdjacencyList_[i][j]; + const TEdge& edge = AdjacencyList_[i][j]; if (i <= edge.Target) { - OS << " " << getTableName(i) << " -- " << getTableName(edge.Target) - << " [label = \"" - << getRelationName(i, edge.ColumnLHS) << " = " - << getRelationName(edge.Target, edge.ColumnRHS) - << "\"];\n"; + os << " " << getTableName(i) << " -- " << getTableName(edge.Target) + << " [label = \"" + << getRelationName(i, edge.ColumnLHS) << " = " + << getRelationName(edge.Target, edge.ColumnRHS) + << "\"];\n"; } } } - OS << "}\n"; + os << "}\n"; } std::vector TRelationGraph::GetDegrees() const { std::vector degrees(AdjacencyList_.size()); - for (unsigned i = 0; i < AdjacencyList_.size(); ++ i) { + for (unsigned i = 0; i < AdjacencyList_.size(); ++i) { degrees[i] = AdjacencyList_[i].size(); } @@ -280,7 +284,7 @@ void TRelationGraph::ReorderDFS() { std::vector newOrder; newOrder.reserve(GetN()); - auto searchDepthFirst = [&](auto &&self, unsigned node) { + auto searchDepthFirst = [&](auto&& self, unsigned node) { if (visited[node]) { return; } @@ -306,9 +310,9 @@ void TRelationGraph::ReorderDFS() { Schema_.Rename(oldToNew); } -void TRelationGraph::Rename(const std::vector &oldToNew) { +void TRelationGraph::Rename(const std::vector& oldToNew) { TAdjacencyList newGraph(GetN()); - for (unsigned u = 0; u < GetN(); ++ u) { + for (unsigned u = 0; u < GetN(); ++u) { for (TEdge edge : AdjacencyList_[u]) { unsigned v = edge.Target; @@ -320,13 +324,26 @@ void TRelationGraph::Rename(const std::vector &oldToNew) { AdjacencyList_ = newGraph; } +ui32 TRelationGraph::GetNumEdges() const { + ui32 numEdges = 0; + for (auto edges : AdjacencyList_) { + numEdges += edges.size(); + } + + return numEdges; +} -TSchemaStats TSchemaStats::MakeRandom(TRNG &mt, const TSchema &schema, unsigned a, unsigned b) { +int TRelationGraph::GetNumComponents() const { + auto comp = FindComponents(); + return comp.empty() ? 0 : *std::max_element(comp.begin(), comp.end()) + 1; +} + +TSchemaStats TSchemaStats::MakeRandom(TRNG& rng, const TSchema& schema, unsigned a, unsigned b) { std::uniform_int_distribution<> distribution(a, b); std::vector stats(schema.GetSize()); - for (unsigned i = 0; i < schema.GetSize(); ++ i) { - unsigned RowSize = std::pow(10, distribution(mt)); + for (unsigned i = 0; i < schema.GetSize(); ++i) { + unsigned RowSize = std::pow(10, distribution(rng)); unsigned ByteSize = RowSize * 64; stats[i] = {ByteSize, RowSize}; } @@ -338,9 +355,10 @@ std::string TSchemaStats::ToJSON() const { std::stringstream ss; ss << "{"; - for (unsigned i = 0; i < Stats_.size(); ++ i) { - if (i != 0) + for (unsigned i = 0; i < Stats_.size(); ++i) { + if (i != 0) { ss << ","; + } ss << "\"" << getTablePath(i) << "\": "; ss << "{"; @@ -353,12 +371,12 @@ std::string TSchemaStats::ToJSON() const { return ss.str(); } -TRelationGraph GenerateLine([[maybe_unused]] TRNG &rng, unsigned numNodes) { +TRelationGraph GeneratePath(unsigned numNodes) { TRelationGraph graph(numNodes); unsigned lastVertex = 0; bool first = true; - for (unsigned i = 0; i < numNodes; ++ i) { + for (unsigned i = 0; i < numNodes; ++i) { if (!first) { graph.Connect(lastVertex, i); } @@ -370,22 +388,22 @@ TRelationGraph GenerateLine([[maybe_unused]] TRNG &rng, unsigned numNodes) { return graph; } -TRelationGraph GenerateStar([[maybe_unused]] TRNG &rng, unsigned numNodes) { +TRelationGraph GenerateStar(unsigned numNodes) { TRelationGraph graph(numNodes); unsigned root = 0; - for (unsigned i = 1; i < numNodes; ++ i) { + for (unsigned i = 1; i < numNodes; ++i) { graph.Connect(root, i); } return graph; } -TRelationGraph GenerateFullyConnected([[maybe_unused]] TRNG &rng, unsigned numNodes) { +TRelationGraph GenerateClique(unsigned numNodes) { TRelationGraph graph(numNodes); - for (unsigned i = 0; i < numNodes; ++ i) { - for (unsigned j = i + 1; j < numNodes; ++ j) { + for (unsigned i = 0; i < numNodes; ++i) { + for (unsigned j = i + 1; j < numNodes; ++j) { graph.Connect(i, j); } } @@ -393,24 +411,63 @@ TRelationGraph GenerateFullyConnected([[maybe_unused]] TRNG &rng, unsigned numNo return graph; } -static std::vector GenerateRandomPruferSequence(TRNG &mt, unsigned numNodes) { - assert(numNodes >= 2); +TRelationGraph GenerateTreeFromPruferSequence(const std::vector& prufer) { + unsigned n = prufer.size() + 2; + + std::vector degree(n, 1); + for (unsigned i : prufer) { + ++ degree[i]; + } + + TRelationGraph graph(n); + + for (unsigned u : prufer) { + for (unsigned v = 0; v < n; ++ v) { + if (degree[v] == 1) { + graph.Connect(u, v); + + -- degree[v]; + -- degree[u]; + break; + } + } + } + + int u = -1; + unsigned v = 0; + for (; v < n; ++ v) { + if (degree[v] == 1) { + if (u != -1) { + graph.Connect(u, v); + break; + } + + u = v; + } + } + + return graph; +} + + +static std::vector GenerateRandomPruferSequence(TRNG& rng, unsigned numNodes) { + Y_ASSERT(numNodes >= 2); std::uniform_int_distribution<> distribution(0, numNodes - 1); std::vector prufer(numNodes - 2); for (unsigned i = 0; i < numNodes - 2; ++i) { - prufer[i] = distribution(mt); + prufer[i] = distribution(rng); } return prufer; } -TRelationGraph GenerateRandomTree(TRNG &mt, unsigned numNodes) { - auto prufer = GenerateRandomPruferSequence(mt, numNodes); - return TRelationGraph::FromPrufer(prufer); +TRelationGraph GenerateRandomTree(TRNG& rng, unsigned numNodes) { + auto prufer = GenerateRandomPruferSequence(rng, numNodes); + return GenerateTreeFromPruferSequence(prufer); } -void NormalizeProbabilities(std::vector& probabilities) { +static void NormalizeProbabilities(std::vector& probabilities) { double sum = std::accumulate(probabilities.begin(), probabilities.end(), 0.0); if (sum > 0.0) { for (double& probability : probabilities) { @@ -430,7 +487,9 @@ std::vector SampleFromPMF(TRNG& rng, const std::vector& probabiliti } std::vector GenerateLogNormalDegrees(TRNG& rng, int numVertices, double mu, double sigma, int minDegree, int maxDegree) { - if (maxDegree == -1) maxDegree = numVertices - 1; + if (maxDegree == -1) { + maxDegree = numVertices - 1; + } std::vector probabilities(maxDegree - minDegree + 1); for (int k = minDegree; k <= maxDegree; k++) { @@ -447,11 +506,12 @@ std::vector GenerateLogNormalDegrees(TRNG& rng, int numVertices, double mu, probabilities[k - minDegree] = (1.0 / (x * sigma * std::sqrt(2.0 * M_PI))) * std::exp(-0.5 * z * z); } + // Not strictly necessary, but makes it easier to inspect probabilities NormalizeProbabilities(probabilities); return SampleFromPMF(rng, probabilities, numVertices, minDegree); } -TRelationGraph GenerateRandomChungLuGraph(TRNG &mt, const std::vector& degrees) { +TRelationGraph GenerateRandomChungLuGraph(TRNG& rng, const std::vector& degrees) { TRelationGraph graph(degrees.size()); double sum = std::accumulate(degrees.begin(), degrees.end(), 0.0); @@ -460,9 +520,9 @@ TRelationGraph GenerateRandomChungLuGraph(TRNG &mt, const std::vector& degr } std::uniform_real_distribution<> distribution(0, 1); - for (ui32 i = 0; i < degrees.size(); ++ i) { - for (ui32 j = i + 1; j < degrees.size(); ++ j) { - if (distribution(mt) < std::min(1.0, degrees[i] * degrees[j] / sum)) { + for (ui32 i = 0; i < degrees.size(); ++i) { + for (ui32 j = i + 1; j < degrees.size(); ++j) { + if (distribution(rng) < std::min(1.0, degrees[i] * degrees[j] / sum)) { graph.Connect(i, j); } } @@ -471,16 +531,15 @@ TRelationGraph GenerateRandomChungLuGraph(TRNG &mt, const std::vector& degr return graph; } - -void MakeEvenSum(std::vector& degrees) { +static void MakeEvenSum(std::vector& degrees) { int sum = std::accumulate(degrees.begin(), degrees.end(), 0); if (sum % 2 == 1) { auto minIt = std::min_element(degrees.begin(), degrees.end()); - ++ *minIt; + ++*minIt; } } -bool SatisfiesErdosGallai(std::vector degrees) { +static bool CheckSatisfiesErdosGallai(std::vector degrees) { int sum = std::accumulate(degrees.begin(), degrees.end(), 0); if (sum % 2 != 0) { @@ -496,11 +555,11 @@ bool SatisfiesErdosGallai(std::vector degrees) { std::sort(degrees.rbegin(), degrees.rend()); uint64_t sumLeft = 0; - for (uint64_t k = 0; k < degrees.size(); ++ k) { + for (uint64_t k = 0; k < degrees.size(); ++k) { sumLeft += degrees[k]; uint64_t sumRight = k * (k + 1); - for (uint64_t i = k + 1; i < degrees.size(); ++ i) { + for (uint64_t i = k + 1; i < degrees.size(); ++i) { sumRight += std::min(k + 1, degrees[i]); } @@ -549,23 +608,23 @@ std::vector MakeGraphicConnected(std::vector degrees) { MakeEvenSum(degrees); int iterations = 0; - while ((!SatisfiesErdosGallai(degrees) || !CanBeConnected(degrees)) && iterations ++ < MAX_ITERATIONS) { + while ((!CheckSatisfiesErdosGallai(degrees) || !CanBeConnected(degrees)) && iterations++ < MAX_ITERATIONS) { std::sort(degrees.begin(), degrees.end(), std::greater()); - if (!SatisfiesErdosGallai(degrees)) { + if (!CheckSatisfiesErdosGallai(degrees)) { // Reduce max, increase min (redistribute) if (degrees[0] > degrees[degrees.size() - 1] + 1) { - -- degrees[0]; - ++ degrees[degrees.size() - 1]; + --degrees[0]; + ++degrees[degrees.size() - 1]; } else { - -- degrees[0]; + --degrees[0]; } } else if (!CanBeConnected(degrees)) { // Increase minimum degrees to help connectivity for (int& degree : degrees) { ui32 edges = std::accumulate(degrees.begin(), degrees.end(), 0) / 2; if (degree < 2 && edges + 2 <= degrees.size() * (degrees.size() - 1) / 2) { - ++ degree; + ++degree; } } } @@ -579,7 +638,7 @@ std::vector MakeGraphicConnected(std::vector degrees) { TRelationGraph ConstructGraphHavelHakimi(std::vector degrees) { TRelationGraph graph(degrees.size()); - std::vector> nodes; + std::vector> nodes; for (uint32_t i = 0; i < degrees.size(); ++i) { nodes.push_back({degrees[i], i}); } @@ -602,24 +661,24 @@ TRelationGraph ConstructGraphHavelHakimi(std::vector degrees) { break; } - for (uint32_t i = 0; i < static_cast(degree); ++ i) { + for (uint32_t i = 0; i < static_cast(degree); ++i) { uint32_t v = nodes[i].second; graph.Connect(u, v); - -- nodes[i].first; + --nodes[i].first; } } return graph; } -void MCMCRandomize(TRNG &mt, TRelationGraph& graph) { +void MCMCRandomize(TRNG& rng, TRelationGraph& graph) { std::uniform_int_distribution<> nodeDist(0, graph.GetN() - 1); - std::uniform_real_distribution<> probDist(0.0, 1.0); + std::uniform_real_distribution<> distribution(0.0, 1.0); - ui32 numEdges = graph.GetEdges(); + ui32 numEdges = graph.GetNumEdges(); ui32 numSwaps = numEdges * log(numEdges); - auto &adjacency = graph.GetAdjacencyList(); + auto& adjacency = graph.GetAdjacencyList(); const double TEMP_START = 5.0; const double TEMP_END = 0.1; @@ -633,34 +692,42 @@ void MCMCRandomize(TRNG &mt, TRelationGraph& graph) { double progress = std::min(1.0, static_cast(attempt) / MAX_ATTEMPTS); double temperature = TEMP_START * std::pow(TEMP_END / TEMP_START, progress); - ++ attempt; + ++attempt; - int a = nodeDist(mt); - if (adjacency[a].empty()) continue; + int a = nodeDist(rng); + if (adjacency[a].empty()) { + continue; + } - int bIdx = std::uniform_int_distribution<>(0, adjacency[a].size() - 1)(mt); + int bIdx = std::uniform_int_distribution<>(0, adjacency[a].size() - 1)(rng); int b = adjacency[a][bIdx].Target; - int c = nodeDist(mt); - if (c == a || c == b || adjacency[c].empty()) continue; + int c = nodeDist(rng); + if (c == a || c == b || adjacency[c].empty()) { + continue; + } - int dIdx = std::uniform_int_distribution<>(0, adjacency[c].size() - 1)(mt); + int dIdx = std::uniform_int_distribution<>(0, adjacency[c].size() - 1)(rng); int d = adjacency[c][dIdx].Target; - if (d == a || d == b || d == c) continue; - if (graph.HasEdge(a, c) || graph.HasEdge(b, d)) continue; + if (d == a || d == b || d == c) { + continue; + } + if (graph.HasEdge(a, c) || graph.HasEdge(b, d)) { + continue; + } - int oldComponents = graph.NumComponents(); + int oldComponents = graph.GetNumComponents(); graph.Disconnect(a, b); graph.Disconnect(c, d); graph.Connect(a, c); graph.Connect(b, d); - int newComponents = graph.NumComponents(); + int newComponents = graph.GetNumComponents(); double deltaEnergy = CONNECTIVITY_PENALTY * (newComponents - oldComponents); - bool accept = probDist(mt) < std::exp(-deltaEnergy / temperature); + bool accept = distribution(rng) < std::exp(-deltaEnergy / temperature); if (accept) { ++ successfulSwaps; @@ -673,4 +740,4 @@ void MCMCRandomize(TRNG &mt, TRelationGraph& graph) { } } -} +} // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_generator.h b/ydb/core/kqp/ut/join/kqp_join_topology_generator.h index d54397c60359..1b7f24b79805 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_generator.h +++ b/ydb/core/kqp/ut/join/kqp_join_topology_generator.h @@ -3,37 +3,34 @@ #include #include -#include -#include +#include #include #include -#include - +#include namespace NKikimr::NKqp { - using TRNG = TSerializableMT19937; - class TLexicographicalNameGenerator { public: - static std::string getName(unsigned ID, bool lowerCase = true) { - if (ID < Base_) - return std::string(1, fromDigit(ID, lowerCase)); + static std::string getName(unsigned id, bool lowerCase = true) { + if (id < Base_) { + return std::string(1, fromDigit(id, lowerCase)); + } - ID -= Base_; + id -= Base_; unsigned count = 1; unsigned step = Base_; - for (; ID >= step;) { - ID -= step; + for (; id >= step;) { + id -= step; step *= step; count *= 2; } std::string result(count, fromDigit(Base_ - 1, lowerCase)); - return result + fromNumber(ID, result.size(), lowerCase); + return result + fromNumber(id, result.size(), lowerCase); } private: @@ -48,30 +45,26 @@ class TLexicographicalNameGenerator { } static char fromDigit(unsigned value, bool lowerCase) { - assert(0 <= value && value < Base_); + Y_ASSERT(0 <= value && value < Base_); return (lowerCase ? 'a' : 'A') + value; } static constexpr unsigned Base_ = 'z' - 'a' + 1; }; - - - struct TPitmanYorConfig { double Alpha; double Theta; - void DumpParamsHeader(IOutputStream &OS) { - OS << "alpha,theta"; + void DumpParamsHeader(IOutputStream& os) { + os << "alpha,theta"; } - void DumpParams(IOutputStream &OS) { - OS << Alpha << "," << Theta; + void DumpParams(IOutputStream& os) { + os << Alpha << "," << Theta; } }; - class TTable { public: TTable(unsigned numColumns = 0) @@ -87,7 +80,6 @@ class TTable { unsigned NumColumns_; }; - class TSchema { public: TSchema(unsigned numNodes) @@ -119,7 +111,6 @@ class TSchema { std::vector Tables_; }; - class TRelationGraph { public: TRelationGraph(unsigned numNodes) @@ -128,105 +119,47 @@ class TRelationGraph { { } - static TRelationGraph FromPrufer(const std::vector& prufer); - void Connect(unsigned lhs, unsigned rhs); + void Disconnect(unsigned u, unsigned v); + bool HasEdge(unsigned u, unsigned v) const; - void Disconnect(unsigned u, unsigned v) { - auto& adjacencyU = AdjacencyList_[u]; - auto& adjacencyV = AdjacencyList_[v]; - adjacencyU.erase(std::remove_if(adjacencyU.begin(), adjacencyU.end(), [v](TEdge edge) { return edge.Target == v; }), adjacencyU.end()); - adjacencyV.erase(std::remove_if(adjacencyV.begin(), adjacencyV.end(), [u](TEdge edge) { return edge.Target == u; }), adjacencyV.end()); - - } - - bool HasEdge(unsigned u, unsigned v) const { - for (ui32 i = 0; i < AdjacencyList_[u].size(); ++i) { - if (AdjacencyList_[u][i].Target == v) { - return true; - } - } - - return false; - } - - std::vector FindComponents() const { - std::vector component(GetN(), -1); - int numComponents = 0; - - for (unsigned start = 0; start < GetN(); ++ start) { - if (component[start] != -1) { - continue; - } - - std::queue queue; - queue.push(start); - component[start] = numComponents; - - while (!queue.empty()) { - unsigned u = queue.front(); - queue.pop(); - for (TEdge edge : AdjacencyList_[u]) { - unsigned v = edge.Target; - if (component[v] == -1) { - component[v] = numComponents; - queue.push(v); - } - } - } - - ++ numComponents; - } - - return component; - } + std::string MakeQuery() const; - int NumComponents() const { - auto comp = FindComponents(); - return comp.empty() ? 0 : *std::max_element(comp.begin(), comp.end()) + 1; + ui32 GetNumEdges() const; + unsigned GetN() const { + return AdjacencyList_.size(); } + std::vector FindComponents() const; + int GetNumComponents() const; bool IsConnected() const { - return NumComponents() == 1; - } - - int NumEdges() const { - int count = 0; - for (const auto& adjacency : AdjacencyList_) { - count += adjacency.size(); - } - return count / 2; + return GetNumComponents() == 1; } - - std::string MakeQuery() const; - - void DumpGraph(IOutputStream &OS) const; - const TSchema& GetSchema() const { return Schema_; } std::vector GetDegrees() const; - unsigned GetN() const { - return AdjacencyList_.size(); - } - + // Reorder in connected order, meaning that first N vertices form + // a connected subgraph if the whole graph is connected. This is used + // to ensure that each JOIN clause only mentions tables that where + // already joined (or FROM clause) void ReorderDFS(); - void Rename(const std::vector &oldToNew); - - void SetupKeysPitmanYor(TRNG &mt, TPitmanYorConfig config); + // Update vertex numbering accroding to oldToNew map, primarily + // used to reorder graph in connected subgraphs-first order. + void Rename(const std::vector& oldToNew); - ui32 GetEdges() { - ui32 numEdges = 0; - for (auto edges : AdjacencyList_) { - numEdges += edges.size(); - } + // Update keys for the whole graph accroding to Pitman-Yor distribution, + // where degree is distributed into clusters and each cluster means + // that that number of edges joins this particular node with the same key + void SetupKeysPitmanYor(TRNG& rng, TPitmanYorConfig config); - return numEdges; - } + // Dump graph in undirected graphviz dot format. Neato is recommended + // for layouting such graphs. + void DumpGraph(IOutputStream& os) const; public: struct TEdge { @@ -246,7 +179,6 @@ class TRelationGraph { TSchema Schema_; }; - class TSchemaStats { public: struct TTableStats { @@ -260,7 +192,7 @@ class TSchemaStats { { } - static TSchemaStats MakeRandom(TRNG &mt, const TSchema &schema, unsigned a, unsigned b); + static TSchemaStats MakeRandom(TRNG& rng, const TSchema& schema, unsigned a, unsigned b); std::string ToJSON() const; @@ -268,31 +200,45 @@ class TSchemaStats { std::vector Stats_; }; +// Basic topologies, this all have fixed node layouts (not random) +TRelationGraph GeneratePath(unsigned numNodes); +TRelationGraph GenerateStar(unsigned numNodes); +TRelationGraph GenerateClique(unsigned numNodes); -void NormalizeProbabilities(std::vector& probabilities); +// Generate a tree from Prufer sequence (each labeled tree has a +// corresponding unique sequence) +TRelationGraph GenerateTreeFromPruferSequence(const std::vector& prufer); +// Uniformly random trees based on random Prufer sequence +TRelationGraph GenerateRandomTree(TRNG& rng, unsigned numNodes); + +// Random graph using Chung Lu model that approximates graph with given degrees +TRelationGraph GenerateRandomChungLuGraph(TRNG& rng, const std::vector& degrees); + +// Sample a degree sequence from a given probability distribution std::vector SampleFromPMF( TRNG& rng, const std::vector& probabilities, int numVertices, int minDegree); +// Sample a degree sequence from lognormal distribution std::vector GenerateLogNormalDegrees( TRNG& rng, int numVertices, - double logMean = 1.0, double logStdDev = 0.5, - int minDegree = 1, int maxDegree = -1 -); - -TRelationGraph ConstructGraphHavelHakimi(std::vector degrees); + double mu = 1.0, double sigma = 0.5, + int minDegree = 1, int maxDegree = -1); +// Adjust degree sequence to make it graphic (realizable by simple graph +// without self-loops and double edges) and check that it's likely possible +// to make a connected graph with that degree sequence +// (athough this property is not guaranteed) std::vector MakeGraphicConnected(std::vector degrees); -void MCMCRandomize(TRNG &mt, TRelationGraph& graph); - -TRelationGraph GenerateLine(TRNG &rng, unsigned numNodes); -TRelationGraph GenerateStar(TRNG &rng, unsigned numNodes); -TRelationGraph GenerateFullyConnected(TRNG &rng, unsigned numNodes); -TRelationGraph GenerateRandomTree(TRNG &mt, unsigned numNodes); +// Deterministically constructs graph for a given degree sequence +TRelationGraph ConstructGraphHavelHakimi(std::vector degrees); -TRelationGraph GenerateRandomChungLuGraph(TRNG &mt, const std::vector& degrees); +// Randomize graph using ~E*log(E) l-switches (preserve degrees of all +// verticies) Uses Metropolis-Hastings based acceptance with annealing (to make +// switching edges ergodic and still produce connected graphs) +void MCMCRandomize(TRNG& rng, TRelationGraph& graph); } // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp index 6845162dafab..00b288bb53f7 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp @@ -1,35 +1,33 @@ -#include -#include -#include -#include -#include -#include +#include "kqp_join_topology_generator.h" + +#include +#include #include #include #include -#include -#include #include - +#include +#include +#include #include #include #include -#include - -#include - -#include "kqp_join_topology_generator.h" +#include +#include +#include +#include +#include +#include -namespace NKikimr { -namespace NKqp { +namespace NKikimr::NKqp { using namespace NYdb; using namespace NYdb::NTable; Y_UNIT_TEST_SUITE(KqpJoinTopology) { - std::optional ExplainQuery(NYdb::NQuery::TSession session, const std::string &query) { + std::optional ExplainQuery(NYdb::NQuery::TSession session, const std::string& query) { auto explainRes = session.ExecuteQuery(query, NYdb::NQuery::TTxControl::NoTx(), NYdb::NQuery::TExecuteQuerySettings().ExecMode(NQuery::EExecMode::Explain) @@ -60,16 +58,15 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { return true; } - void JustPrintPlan(const TString &plan) { - NYdb::NConsoleClient::TQueryPlanPrinter queryPlanPrinter( - NYdb::NConsoleClient::EDataFormat::PrettyTable, - /*analyzeMode=*/true, Cout, /*maxWidth=*/0 - ); + void JustPrintPlan(const TString& plan) { + NYdb::NConsoleClient::TQueryPlanPrinter queryPlanPrinter( + NYdb::NConsoleClient::EDataFormat::PrettyTable, + /*analyzeMode=*/true, Cout, /*maxWidth=*/0); - queryPlanPrinter.Print(plan); + queryPlanPrinter.Print(plan); } - std::string ConfigureQuery(const std::string &query, bool enableShuffleElimination = false, unsigned optLevel = 2) { + std::string ConfigureQuery(const std::string& query, bool enableShuffleElimination = false, unsigned optLevel = 2) { std::string queryWithShuffleElimination = "PRAGMA ydb.OptShuffleElimination=\""; queryWithShuffleElimination += enableShuffleElimination ? "true" : "false"; queryWithShuffleElimination += "\";\n"; @@ -97,7 +94,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { return std::nullopt; } - assert(savedPlan); + Y_ASSERT(savedPlan); JustPrintPlan(*savedPlan); Cout << "--------------------------------------------------------------------------\n"; @@ -146,7 +143,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { std::optional withShuffleElimination; if (resultType.contains("SE")) { Cout << "--------------------------------- CBO+SE ---------------------------------\n"; - withShuffleElimination = BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/true, /*optLevel=*/2)); + withShuffleElimination = BenchmarkExplain(config, session, ConfigureQuery(query, /*enableShuffleElimination=*/true, /*optLevel=*/2)); if (resultType.contains("0")) { results.emplace("SE-0", (*withShuffleElimination - *withoutCBO).Filter([](double value) { return value > 0; })); @@ -181,7 +178,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { std::unique_ptr GetCBOTestsYDB(TString stats, TDuration compilationTimeout) { TVector settings; - assert(!stats.empty()); + Y_ASSERT(!stats.empty()); NKikimrKqp::TKqpSetting setting; setting.SetName("OptOverrideStatistics"); @@ -233,7 +230,6 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { return BenchmarkShuffleElimination(config, session, resultType, query); } - template void OverrideWithArg(std::string key, TArgs args, auto& value) { if (args.HasArg(key)) { @@ -241,7 +237,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { } } - void OverrideRepeatedTestConfig(std::string prefix, TArgs args, TRepeatedTestConfig &config) { + void OverrideRepeatedTestConfig(std::string prefix, TArgs args, TRepeatedTestConfig& config) { OverrideWithArg(prefix + ".MinRepeats", args, config.MinRepeats); OverrideWithArg(prefix + ".MaxRepeats", args, config.MaxRepeats); OverrideWithArg(prefix + ".Timeout", args, config.Timeout); @@ -273,44 +269,23 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { return config; } - void DumpBenchmarkConfig(IOutputStream &OS, TBenchmarkConfig config) { - OS << "config = {\n"; - OS << " .Warmup = {\n"; - OS << " .MinRepeats = " << config.Warmup.MinRepeats << ",\n"; - OS << " .MaxRepeats = " << config.Warmup.MaxRepeats << ",\n"; - OS << " .Timeout = " << TimeFormatter::Format(config.Warmup.Timeout) << "\n"; - OS << " },\n\n"; - OS << " .Bench = {\n"; - OS << " .MinRepeats = " << config.Bench.MinRepeats << ",\n"; - OS << " .MaxRepeats = " << config.Bench.MaxRepeats << ",\n"; - OS << " .Timeout = " << TimeFormatter::Format(config.Bench.Timeout) << "\n"; - OS << " },\n\n"; - OS << " .SingleRunTimeout = " << TimeFormatter::Format(config.SingleRunTimeout) << ",\n"; - OS << " .MADThreshold = " << config.MADThreshold << "\n"; - OS << "}\n"; + void DumpBenchmarkConfig(IOutputStream& os, TBenchmarkConfig config) { + os << "config = {\n"; + os << " .Warmup = {\n"; + os << " .MinRepeats = " << config.Warmup.MinRepeats << ",\n"; + os << " .MaxRepeats = " << config.Warmup.MaxRepeats << ",\n"; + os << " .Timeout = " << TimeFormatter::Format(config.Warmup.Timeout) << "\n"; + os << " },\n\n"; + os << " .Bench = {\n"; + os << " .MinRepeats = " << config.Bench.MinRepeats << ",\n"; + os << " .MaxRepeats = " << config.Bench.MaxRepeats << ",\n"; + os << " .Timeout = " << TimeFormatter::Format(config.Bench.Timeout) << "\n"; + os << " },\n\n"; + os << " .SingleRunTimeout = " << TimeFormatter::Format(config.SingleRunTimeout) << ",\n"; + os << " .MADThreshold = " << config.MADThreshold << "\n"; + os << "}\n"; } - - class TDegreeDistributionGenerator { - public: - virtual std::vector Initialize(TArgs args) = 0; - virtual std::vector GenerateDegreeSequence(TRNG &rng) = 0; - - virtual ~TDegreeDistributionGenerator() = default; - }; - - class TTopologyTester { - public: - virtual void Initialize(TArgs args) = 0; - virtual TRelationGraph ProduceGraph(TRNG &rng) = 0; - virtual void DumpParamsHeader(IOutputStream &OS) = 0; - virtual void DumpParams(IOutputStream &OS) = 0; - virtual void Loop(std::function) {} - - virtual ~TTopologyTester() = default; - }; - - TPitmanYorConfig GetPitmanYorConfig(TArgs args) { return TPitmanYorConfig{ .Alpha = args.GetArgOrDefault("alpha", "0.5").GetValue(), @@ -337,7 +312,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { static TBenchState fromHex(const std::string& hex) { TBenchState state; ui32* parts[] = {&state.Seed, &state.TopologyCounter, &state.MCMCCounter, &state.KeyCounter}; - for (ui32 i = 0; i < 4; ++ i) { + for (ui32 i = 0; i < Y_ARRAY_SIZE(parts); ++ i) { *parts[i] = std::stoul(hex.substr(i * 8, 8), nullptr, 16); } @@ -355,7 +330,6 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { std::string OutputDir; std::map Streams = {}; - }; TTestContext CreateTestContext(TArgs args, std::string outputDir = "") { @@ -377,10 +351,10 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { auto db = kikimr->GetQueryClient(); auto session = db.GetSession().GetValueSync().GetSession(); - return { std::move(kikimr), std::move(db), std::move(session), state, std::move(rng), outputDir }; + return {std::move(kikimr), std::move(db), std::move(session), state, std::move(rng), outputDir}; } - std::string WriteGraph(TTestContext &ctx, ui32 graphID, const TRelationGraph &graph) { + std::string WriteGraph(TTestContext& ctx, ui32 graphID, const TRelationGraph& graph) { if (ctx.OutputDir.empty()) { return ""; } @@ -399,10 +373,9 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { return graphName; } - void WriteAllStats(TTestContext &ctx, const std::string& prefix, + void WriteAllStats(TTestContext& ctx, const std::string& prefix, const std::string& header, const std::string& params, const std::map& stats) { - if (ctx.OutputDir.empty()) { return; } @@ -411,16 +384,16 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { std::filesystem::create_directories(ctx.OutputDir); } - for (const auto &[key, stat]: stats) { + for (const auto& [key, stat] : stats) { std::string name = prefix + key; if (!ctx.Streams.contains(name)) { auto filename = TString(ctx.OutputDir + "/" + name + ".csv"); - auto &os = ctx.Streams.emplace(name, TUnbufferedFileOutput(filename)).first->second; + auto& os = ctx.Streams.emplace(name, TUnbufferedFileOutput(filename)).first->second; os << header << "\n"; } - auto &os = ctx.Streams.find(name)->second; + auto& os = ctx.Streams.find(name)->second; os << params << ","; stat.ComputeStatistics().ToCSV(os); @@ -430,7 +403,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { void AccumulateAllStats(std::map& cummulative, const std::map& stats) { - for (auto &[key, stat] : stats) { + for (auto& [key, stat] : stats) { if (!cummulative.contains(key)) { cummulative.emplace(key, stat); } @@ -439,12 +412,11 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { } } - - using TTopologyFunction = TRelationGraph (*) (TRNG rng, ui32 n, double mu, double sigma); + using TTopologyFunction = TRelationGraph (*)(TRNG& rng, ui32 n, double mu, double sigma); template TTopologyFunction GetTrivialTopology() { - return []([[maybe_unused]] TRNG rng, ui32 n, [[maybe_unused]] double mu, [[maybe_unused]] double sigma) { - return TTrivialTopologyGenerator(rng, n); + return []([[maybe_unused]] TRNG& rng, ui32 n, [[maybe_unused]] double mu, [[maybe_unused]] double sigma) { + return TTrivialTopologyGenerator(n); }; } @@ -454,19 +426,21 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { } if (topologyName == "path") { - return GetTrivialTopology(); + return GetTrivialTopology(); } if (topologyName == "clique") { - return GetTrivialTopology(); + return GetTrivialTopology(); } if (topologyName == "random-tree") { - return GetTrivialTopology(); + return []([[maybe_unused]] TRNG& rng, ui32 n, [[maybe_unused]] double mu, [[maybe_unused]] double sigma) { + return GenerateRandomTree(rng, n); + }; } if (topologyName == "mcmc") { - return []([[maybe_unused]] TRNG rng, ui32 n, [[maybe_unused]] double mu, [[maybe_unused]] double sigma) { + return []([[maybe_unused]] TRNG& rng, ui32 n, [[maybe_unused]] double mu, [[maybe_unused]] double sigma) { Cout << "================================= METRICS ================================\n"; auto sampledDegrees = GenerateLogNormalDegrees(rng, n, mu, sigma); Cout << "sampled degrees: " << JoinSeq(", ", sampledDegrees) << "\n"; @@ -480,7 +454,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { } if (topologyName == "chung-lu") { - return []([[maybe_unused]] TRNG rng, ui32 n, [[maybe_unused]] double mu, [[maybe_unused]] double sigma) { + return []([[maybe_unused]] TRNG& rng, ui32 n, [[maybe_unused]] double mu, [[maybe_unused]] double sigma) { Cout << "================================= METRICS ================================\n"; auto initialDegrees = GenerateLogNormalDegrees(rng, n, mu, sigma); Cout << "initial degrees: " << JoinSeq(", ", initialDegrees) << "\n"; @@ -493,7 +467,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { throw std::runtime_error("Unknown topology: '" + topologyName + "'"); } - void RunBenches(TTestContext &ctx, TBenchmarkConfig config, TArgs args) { + void RunBenches(TTestContext& ctx, TBenchmarkConfig config, TArgs args) { std::string resultType = args.GetStringOrDefault("result", "SE"); ui64 topologyGenerationRepeats = args.GetArgOrDefault("gen-n", "1").GetValue(); @@ -587,18 +561,15 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { } } - WriteAllStats(ctx, "aggregate-", headerAggregate, commonParams.str(), aggregate); } - stop:; + stop:; } } } } } - - Y_UNIT_TEST(Benchmark) { TArgs args{GetTestParam("TOPOLOGY")}; if (!args.HasArg("N")) { @@ -614,7 +585,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { RunBenches(ctx, config, args); } - void Check(TRunningStatistics& stats, ui32 n, double min, double max, double median, double mad, double q1, double q3, double iqr, double mean, double stdev) { + void CheckStats(TRunningStatistics& stats, ui32 n, double min, double max, double median, double mad, double q1, double q3, double iqr, double mean, double stdev) { const double TOLERANCE = 0.0001; UNIT_ASSERT_EQUAL(stats.GetN(), n); @@ -641,79 +612,77 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { UNIT_ASSERT_DOUBLES_EQUAL(report.Stdev, stdev, TOLERANCE); } - Y_UNIT_TEST(TStatistics) { TRunningStatistics stats; stats.AddValue(1); stats.AddValue(1); - Check(stats, 2, 1, 1, 1, 0, 1, 1, 0, 1, 0); + CheckStats(stats, 2, 1, 1, 1, 0, 1, 1, 0, 1, 0); stats.AddValue(1); - Check(stats, 3, 1, 1, 1, 0, 1, 1, 0, 1, 0); + CheckStats(stats, 3, 1, 1, 1, 0, 1, 1, 0, 1, 0); stats.AddValue(1); - Check(stats, 4, 1, 1, 1, 0, 1, 1, 0, 1, 0); + CheckStats(stats, 4, 1, 1, 1, 0, 1, 1, 0, 1, 0); stats.AddValue(3); - Check(stats, 5, 1, 3, 1, 0, 1, 1, 0, 1.4000, 0.8944); + CheckStats(stats, 5, 1, 3, 1, 0, 1, 1, 0, 1.4000, 0.8944); stats.AddValue(5); - Check(stats, 6, 1, 5, 1, 0, 1, 2.5000, 1.5000, 2, 1.6733); + CheckStats(stats, 6, 1, 5, 1, 0, 1, 2.5000, 1.5000, 2, 1.6733); stats.AddValue(7); - Check(stats, 7, 1, 7, 1, 0, 1, 4, 3, 2.7143, 2.4300); + CheckStats(stats, 7, 1, 7, 1, 0, 1, 4, 3, 2.7143, 2.4300); stats.AddValue(7); - Check(stats, 8, 1, 7, 2, 1, 1, 5.5000, 4.5000, 3.2500, 2.7124); + CheckStats(stats, 8, 1, 7, 2, 1, 1, 5.5000, 4.5000, 3.2500, 2.7124); stats.AddValue(8); - Check(stats, 9, 1, 8, 3, 2, 1, 7, 6, 3.7778, 2.9907); + CheckStats(stats, 9, 1, 8, 3, 2, 1, 7, 6, 3.7778, 2.9907); stats.AddValue(100); - Check(stats, 10, 1, 100, 4, 3, 1, 7, 6, 13.4000, 30.5585); + CheckStats(stats, 10, 1, 100, 4, 3, 1, 7, 6, 13.4000, 30.5585); stats.AddValue(12); - Check(stats, 11, 1, 100, 5, 4, 1, 7.5000, 6.5000, 13.2727, 28.9934); + CheckStats(stats, 11, 1, 100, 5, 4, 1, 7.5000, 6.5000, 13.2727, 28.9934); stats.AddValue(15); - Check(stats, 12, 1, 100, 6, 5, 1, 9, 8, 13.4167, 27.6486); + CheckStats(stats, 12, 1, 100, 6, 5, 1, 9, 8, 13.4167, 27.6486); stats.AddValue(11); - Check(stats, 13, 1, 100, 7, 5, 1, 11, 10, 13.2308, 26.4800); + CheckStats(stats, 13, 1, 100, 7, 5, 1, 11, 10, 13.2308, 26.4800); stats.AddValue(18); - Check(stats, 14, 1, 100, 7, 5.5000, 1.5000, 11.7500, 10.2500, 13.5714, 25.4731); + CheckStats(stats, 14, 1, 100, 7, 5.5000, 1.5000, 11.7500, 10.2500, 13.5714, 25.4731); stats.AddValue(19); - Check(stats, 15, 1, 100, 7, 6, 2, 13.5000, 11.5000, 13.9333, 24.5865); + CheckStats(stats, 15, 1, 100, 7, 6, 2, 13.5000, 11.5000, 13.9333, 24.5865); stats.AddValue(14); - Check(stats, 16, 1, 100, 7.5000, 6.5000, 2.5000, 14.2500, 11.7500, 13.9375, 23.7528); + CheckStats(stats, 16, 1, 100, 7.5000, 6.5000, 2.5000, 14.2500, 11.7500, 13.9375, 23.7528); stats.AddValue(21); - Check(stats, 17, 1, 100, 8, 7, 3, 15, 12, 14.3529, 23.0623); + CheckStats(stats, 17, 1, 100, 8, 7, 3, 15, 12, 14.3529, 23.0623); stats.AddValue(9); - Check(stats, 18, 1, 100, 8.5000, 6, 3.5000, 14.7500, 11.2500, 14.0556, 22.4092); + CheckStats(stats, 18, 1, 100, 8.5000, 6, 3.5000, 14.7500, 11.2500, 14.0556, 22.4092); stats.AddValue(25); - Check(stats, 19, 1, 100, 9, 6, 4, 16.5000, 12.5000, 14.6316, 21.9221); + CheckStats(stats, 19, 1, 100, 9, 6, 4, 16.5000, 12.5000, 14.6316, 21.9221); stats.AddValue(17); - Check(stats, 20, 1, 100, 10, 7, 4.5000, 17.2500, 12.7500, 14.7500, 21.3440); + CheckStats(stats, 20, 1, 100, 10, 7, 4.5000, 17.2500, 12.7500, 14.7500, 21.3440); stats.AddValue(18); - Check(stats, 21, 1, 100, 11, 7, 5, 18, 13, 14.9048, 20.8156); + CheckStats(stats, 21, 1, 100, 11, 7, 5, 18, 13, 14.9048, 20.8156); stats.AddValue(10); - Check(stats, 22, 1, 100, 10.5000, 7, 5.5000, 17.7500, 12.2500, 14.6818, 20.3409); + CheckStats(stats, 22, 1, 100, 10.5000, 7, 5.5000, 17.7500, 12.2500, 14.6818, 20.3409); stats.AddValue(22); - Check(stats, 23, 1, 100, 11, 7, 6, 18, 12, 15, 19.9317); + CheckStats(stats, 23, 1, 100, 11, 7, 6, 18, 12, 15, 19.9317); } -} +} // Y_UNIT_TEST_SUITE(KqpJoinTopology) -} -} +} // namespace NKikimr::NKqp diff --git a/ydb/library/yql/dq/opt/dq_opt_join_cbo_factory.cpp b/ydb/library/yql/dq/opt/dq_opt_join_cbo_factory.cpp index e19bb3f6b9f0..46f6bba25420 100644 --- a/ydb/library/yql/dq/opt/dq_opt_join_cbo_factory.cpp +++ b/ydb/library/yql/dq/opt/dq_opt_join_cbo_factory.cpp @@ -9,7 +9,7 @@ namespace NYql::NDq { namespace { class TDqOptimizerFactory : public IOptimizerFactory { public: - virtual IOptimizerNew::TPtr MakeJoinCostBasedOptimizerNative(IProviderContext& pctx, TExprContext& ectx, const TOptimizerSettings& settings) const override { + virtual IOptimizerNew::TPtr MakeJoinCostBasedOptimizerNative(IProviderContext& pctx, TExprContext& ectx, const TCBOSettings& settings) const override { return IOptimizerNew::TPtr(MakeNativeOptimizerNew(pctx, settings, ectx, false, nullptr)); } diff --git a/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp b/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp index 50d66c083133..9d24f75ce534 100644 --- a/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp +++ b/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp @@ -314,7 +314,7 @@ class TOptimizerNativeNew: public IOptimizerNew { public: TOptimizerNativeNew( IProviderContext& ctx, - const TOptimizerSettings &optimizerSettings, + const TCBOSettings& optimizerSettings, TExprContext& exprCtx, bool enableShuffleElimination, TSimpleSharedPtr orderingsFSM, @@ -506,7 +506,7 @@ class TOptimizerNativeNew: public IOptimizerNew { } private: - TOptimizerSettings OptimizerSettings_; + TCBOSettings OptimizerSettings_; TExprContext& ExprCtx; bool EnableShuffleElimination; @@ -516,7 +516,7 @@ class TOptimizerNativeNew: public IOptimizerNew { IOptimizerNew* MakeNativeOptimizerNew( IProviderContext& pctx, - const TOptimizerSettings &settings, + const TCBOSettings& settings, TExprContext& ectx, bool enableShuffleElimination, TSimpleSharedPtr orderingsFSM, diff --git a/ydb/library/yql/dq/opt/dq_opt_join_cost_based.h b/ydb/library/yql/dq/opt/dq_opt_join_cost_based.h index 448d1dbb2a0d..09b8cc919387 100644 --- a/ydb/library/yql/dq/opt/dq_opt_join_cost_based.h +++ b/ydb/library/yql/dq/opt/dq_opt_join_cost_based.h @@ -50,7 +50,7 @@ void CollectInterestingOrderingsFromJoinTree( IOptimizerNew* MakeNativeOptimizerNew( IProviderContext& ctx, - const TOptimizerSettings &settings, + const TCBOSettings& settings, TExprContext& ectx, bool enableShuffleElimination, TSimpleSharedPtr orderingsFSM = nullptr, diff --git a/ydb/library/yql/dq/opt/ut/dq_cbo_ut.cpp b/ydb/library/yql/dq/opt/ut/dq_cbo_ut.cpp index f50eeb9120fd..6b08d5e9cc24 100644 --- a/ydb/library/yql/dq/opt/ut/dq_cbo_ut.cpp +++ b/ydb/library/yql/dq/opt/ut/dq_cbo_ut.cpp @@ -35,7 +35,7 @@ Y_UNIT_TEST(Empty) { TBaseProviderContext pctx; TExprContext dummyCtx; - TOptimizerSettings settings{}; + TCBOSettings settings{}; std::unique_ptr optimizer = std::unique_ptr(MakeNativeOptimizerNew(pctx, settings, dummyCtx, false)); } @@ -43,7 +43,7 @@ Y_UNIT_TEST(JoinSearch2Rels) { TBaseProviderContext pctx; TExprContext dummyCtx; - TOptimizerSettings settings{}; + TCBOSettings settings{}; std::unique_ptr optimizer = std::unique_ptr(MakeNativeOptimizerNew(pctx, settings, dummyCtx, false)); auto rel1 = std::make_shared( @@ -79,7 +79,7 @@ Y_UNIT_TEST(JoinSearch3Rels) { TBaseProviderContext pctx; TExprContext dummyCtx; - TOptimizerSettings settings{}; + TCBOSettings settings{}; std::unique_ptr optimizer = std::unique_ptr(MakeNativeOptimizerNew(pctx, settings, dummyCtx, false)); auto rel1 = std::make_shared("a", @@ -128,7 +128,7 @@ Y_UNIT_TEST(JoinSearchYQL19363) { TBaseProviderContext pctx; TExprContext dummyCtx; - TOptimizerSettings settings{}; + TCBOSettings settings{}; std::unique_ptr optimizer = std::unique_ptr(MakeNativeOptimizerNew(pctx, settings, dummyCtx, false)); TString relName1 = "a,b.c"; @@ -237,7 +237,7 @@ Y_UNIT_TEST(JoinSearchYT24403) { TMockProviderContextYT24403 pctx; TExprContext dummyCtx; - TOptimizerSettings settings{}; + TCBOSettings settings{}; std::unique_ptr optimizer = std::unique_ptr(MakeNativeOptimizerNew(pctx, settings, dummyCtx, false)); const TString relName1 = "a"; @@ -378,7 +378,7 @@ Y_UNIT_TEST(DqOptimizeEquiJoinWithCostsNative) { TExprContext ctx; TBaseProviderContext pctx; std::function optFactory = [&]() { - TOptimizerSettings settings{}; + TCBOSettings settings{}; return MakeNativeOptimizerNew(pctx, settings, ctx, false); }; _DqOptimizeEquiJoinWithCosts(optFactory, ctx); diff --git a/ydb/library/yql/dq/opt/ut/dq_opt_hypergraph_ut.cpp b/ydb/library/yql/dq/opt/ut/dq_opt_hypergraph_ut.cpp index f3ffbc9b92ff..1c1795fd0fb6 100644 --- a/ydb/library/yql/dq/opt/ut/dq_opt_hypergraph_ut.cpp +++ b/ydb/library/yql/dq/opt/ut/dq_opt_hypergraph_ut.cpp @@ -53,7 +53,7 @@ std::shared_ptr Enumerate(const std::shared_ptr(MakeNativeOptimizerNew(ctx, settings, dummyCtx, false)); diff --git a/yql/essentials/core/cbo/cbo_optimizer_new.h b/yql/essentials/core/cbo/cbo_optimizer_new.h index 362be2720bfc..dec5ad427f44 100644 --- a/yql/essentials/core/cbo/cbo_optimizer_new.h +++ b/yql/essentials/core/cbo/cbo_optimizer_new.h @@ -385,7 +385,7 @@ struct IOptimizerNew { struct TExprContext; -struct TOptimizerSettings { +struct TCBOSettings { ui32 MaxDPhypDPTableSize = 100000; ui32 ShuffleEliminationJoinNumCutoff = 14; }; @@ -397,7 +397,7 @@ class IOptimizerFactory : private TNonCopyable { virtual ~IOptimizerFactory() = default; - virtual IOptimizerNew::TPtr MakeJoinCostBasedOptimizerNative(IProviderContext& pctx, TExprContext& ctx, const TOptimizerSettings& settings) const = 0; + virtual IOptimizerNew::TPtr MakeJoinCostBasedOptimizerNative(IProviderContext& pctx, TExprContext& ctx, const TCBOSettings& settings) const = 0; struct TPGSettings { TLogger Logger = [](const TString&) {}; diff --git a/yql/essentials/core/cbo/simple/cbo_simple.cpp b/yql/essentials/core/cbo/simple/cbo_simple.cpp index e729626283e3..0543c0ab7923 100644 --- a/yql/essentials/core/cbo/simple/cbo_simple.cpp +++ b/yql/essentials/core/cbo/simple/cbo_simple.cpp @@ -9,7 +9,7 @@ namespace { class TSimpleOptimizerFactory: public IOptimizerFactory { public: - virtual IOptimizerNew::TPtr MakeJoinCostBasedOptimizerNative(IProviderContext& pctx, TExprContext& ctx, const TOptimizerSettings& settings) const override { + virtual IOptimizerNew::TPtr MakeJoinCostBasedOptimizerNative(IProviderContext& pctx, TExprContext& ctx, const TCBOSettings& settings) const override { Y_UNUSED(pctx); Y_UNUSED(ctx); Y_UNUSED(settings); From 8f8a0006467993cb1e8d374b11b47c2786383d97 Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Tue, 11 Nov 2025 09:08:51 +0000 Subject: [PATCH 40/43] Make Arcadia style checker happy --- yql/essentials/core/cbo/cbo_optimizer_new.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yql/essentials/core/cbo/cbo_optimizer_new.h b/yql/essentials/core/cbo/cbo_optimizer_new.h index dec5ad427f44..232ebd9addae 100644 --- a/yql/essentials/core/cbo/cbo_optimizer_new.h +++ b/yql/essentials/core/cbo/cbo_optimizer_new.h @@ -390,7 +390,7 @@ struct TCBOSettings { ui32 ShuffleEliminationJoinNumCutoff = 14; }; -class IOptimizerFactory : private TNonCopyable { +class IOptimizerFactory: private TNonCopyable { public: using TPtr = std::shared_ptr; using TLogger = std::function; From ed39ebb4a223dcbd4720c23d9d0b7b7a77a878c9 Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Tue, 11 Nov 2025 13:12:03 +0000 Subject: [PATCH 41/43] Better repro message in topology benchmarks --- ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp index 00b288bb53f7..6b4a5ae9cd81 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp @@ -527,14 +527,20 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { std::string state = TBenchState(ctx.RNG.GetSeed(), counterTopology, counterMCMC, counterKeys).toHex(); - Cout << "\n\nReproduce: TOPOLOGY='"; + Cout << "\n\n"; + Cout << "Test #" << idx << "\n"; + Cout << "Reproduce: TOPOLOGY='"; Cout << "type=" << topologyName << "; " - << "N=" << n << "; " - << "alpha=" << alpha << "; " - << "theta=" << theta << "; " - << "sigma=" << sigma << "; " - << "mu=" << mu << "; " - << "state=" << state << "'\n"; + << "N=" << n << "; " + << "alpha=" << alpha << "; " + << "theta=" << theta << "; "; + + if (topologyName == "mcmc" || topologyName == "chung-lu") { + Cout << "sigma=" << sigma << "; " + << "mu=" << mu << "; "; + } + + Cout << "state=" << state << "'\n"; try { auto result = BenchmarkShuffleEliminationOnTopology(config, ctx.Session, resultType, graph); From 60b3623d18317fd61e3c263d8858065ece4a6dda Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Tue, 11 Nov 2025 13:12:32 +0000 Subject: [PATCH 42/43] Reorder option for reproducing bugs with cross join elimination --- ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp index 6b4a5ae9cd81..0eb19a994f03 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp @@ -199,11 +199,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { std::optional> BenchmarkShuffleEliminationOnTopology(TBenchmarkConfig config, NYdb::NQuery::TSession session, std::string resultType, TRelationGraph graph) { - Cout << "================================= CREATE =================================\n"; - graph.DumpGraph(Cout); - - Cout << "================================= REORDER ================================\n"; - graph.ReorderDFS(); + Cout << "================================= GRAPH ==================================\n"; graph.DumpGraph(Cout); Cout << "================================= PREPARE ================================\n"; @@ -473,6 +469,7 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { ui64 topologyGenerationRepeats = args.GetArgOrDefault("gen-n", "1").GetValue(); ui64 mcmcRepeats = args.GetArgOrDefault("mcmc-n", "1").GetValue(); ui64 equiJoinKeysGenerationRepeats = args.GetArgOrDefault("keys-n", "1").GetValue(); + bool reorder = args.GetArgOrDefault("reorder", "1").GetValue() != 0; std::string topologyName = args.GetStringOrDefault("type", "star"); auto generateTopology = GetTopology(topologyName); @@ -540,9 +537,19 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { << "mu=" << mu << "; "; } + if (args.HasArg("reorder")) { + Cout << "reorder=" << reorder << "; "; + } + Cout << "state=" << state << "'\n"; try { + if (reorder) { + Cout << "================================= GRAPH BEFORE REODERING =================\n"; + graph.DumpGraph(Cout); + graph.ReorderDFS(); + } + auto result = BenchmarkShuffleEliminationOnTopology(config, ctx.Session, resultType, graph); if (!result) { goto stop; From 83bb78befe80c5fe67d1543e05ce94e6f237c938 Mon Sep 17 00:00:00 2001 From: alexpaniman Date: Tue, 11 Nov 2025 13:38:49 +0000 Subject: [PATCH 43/43] Extract tests for kqp benches from topology-based benchmarks --- ydb/core/kqp/ut/join/kqp_benches_ut.cpp | 115 ++++++++++++++++++ ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp | 98 --------------- ydb/core/kqp/ut/join/ya.make | 1 + 3 files changed, 116 insertions(+), 98 deletions(-) create mode 100644 ydb/core/kqp/ut/join/kqp_benches_ut.cpp diff --git a/ydb/core/kqp/ut/join/kqp_benches_ut.cpp b/ydb/core/kqp/ut/join/kqp_benches_ut.cpp new file mode 100644 index 000000000000..a4f7fb20cef3 --- /dev/null +++ b/ydb/core/kqp/ut/join/kqp_benches_ut.cpp @@ -0,0 +1,115 @@ +#include "kqp_join_topology_generator.h" + +#include +#include + +namespace NKikimr::NKqp { + +Y_UNIT_TEST_SUITE(KqpBenches) { + + void CheckStats(TRunningStatistics& stats, + ui32 n, double min, double max, + double median, double mad, + double q1, double q3, double iqr, + double mean, double stdev) { + + const double TOLERANCE = 0.0001; + + UNIT_ASSERT_EQUAL(stats.GetN(), n); + UNIT_ASSERT_DOUBLES_EQUAL(stats.GetMin(), min, TOLERANCE); + UNIT_ASSERT_DOUBLES_EQUAL(stats.GetMax(), max, TOLERANCE); + UNIT_ASSERT_DOUBLES_EQUAL(stats.GetMedian(), median, TOLERANCE); + UNIT_ASSERT_DOUBLES_EQUAL(stats.CalculateMAD(), mad, TOLERANCE); + + auto statistics = stats.GetStatistics(); + UNIT_ASSERT_EQUAL(statistics.GetN(), n); + UNIT_ASSERT_DOUBLES_EQUAL(statistics.GetMin(), min, TOLERANCE); + UNIT_ASSERT_DOUBLES_EQUAL(statistics.GetMax(), max, TOLERANCE); + + auto report = statistics.ComputeStatistics(); + UNIT_ASSERT_EQUAL(report.N, n); + UNIT_ASSERT_DOUBLES_EQUAL(report.Min, min, TOLERANCE); + UNIT_ASSERT_DOUBLES_EQUAL(report.Max, max, TOLERANCE); + UNIT_ASSERT_DOUBLES_EQUAL(report.Median, median, TOLERANCE); + UNIT_ASSERT_DOUBLES_EQUAL(report.MAD, mad, TOLERANCE); + UNIT_ASSERT_DOUBLES_EQUAL(report.Q1, q1, TOLERANCE); + UNIT_ASSERT_DOUBLES_EQUAL(report.Q3, q3, TOLERANCE); + UNIT_ASSERT_DOUBLES_EQUAL(report.IQR, iqr, TOLERANCE); + UNIT_ASSERT_DOUBLES_EQUAL(report.Mean, mean, TOLERANCE); + UNIT_ASSERT_DOUBLES_EQUAL(report.Stdev, stdev, TOLERANCE); + } + + Y_UNIT_TEST(TStatistics) { + TRunningStatistics stats; + + stats.AddValue(1); + stats.AddValue(1); + CheckStats(stats, 2, 1, 1, 1, 0, 1, 1, 0, 1, 0); + + stats.AddValue(1); + CheckStats(stats, 3, 1, 1, 1, 0, 1, 1, 0, 1, 0); + + stats.AddValue(1); + CheckStats(stats, 4, 1, 1, 1, 0, 1, 1, 0, 1, 0); + + stats.AddValue(3); + CheckStats(stats, 5, 1, 3, 1, 0, 1, 1, 0, 1.4000, 0.8944); + + stats.AddValue(5); + CheckStats(stats, 6, 1, 5, 1, 0, 1, 2.5000, 1.5000, 2, 1.6733); + + stats.AddValue(7); + CheckStats(stats, 7, 1, 7, 1, 0, 1, 4, 3, 2.7143, 2.4300); + + stats.AddValue(7); + CheckStats(stats, 8, 1, 7, 2, 1, 1, 5.5000, 4.5000, 3.2500, 2.7124); + + stats.AddValue(8); + CheckStats(stats, 9, 1, 8, 3, 2, 1, 7, 6, 3.7778, 2.9907); + + stats.AddValue(100); + CheckStats(stats, 10, 1, 100, 4, 3, 1, 7, 6, 13.4000, 30.5585); + + stats.AddValue(12); + CheckStats(stats, 11, 1, 100, 5, 4, 1, 7.5000, 6.5000, 13.2727, 28.9934); + + stats.AddValue(15); + CheckStats(stats, 12, 1, 100, 6, 5, 1, 9, 8, 13.4167, 27.6486); + + stats.AddValue(11); + CheckStats(stats, 13, 1, 100, 7, 5, 1, 11, 10, 13.2308, 26.4800); + + stats.AddValue(18); + CheckStats(stats, 14, 1, 100, 7, 5.5000, 1.5000, 11.7500, 10.2500, 13.5714, 25.4731); + + stats.AddValue(19); + CheckStats(stats, 15, 1, 100, 7, 6, 2, 13.5000, 11.5000, 13.9333, 24.5865); + + stats.AddValue(14); + CheckStats(stats, 16, 1, 100, 7.5000, 6.5000, 2.5000, 14.2500, 11.7500, 13.9375, 23.7528); + + stats.AddValue(21); + CheckStats(stats, 17, 1, 100, 8, 7, 3, 15, 12, 14.3529, 23.0623); + + stats.AddValue(9); + CheckStats(stats, 18, 1, 100, 8.5000, 6, 3.5000, 14.7500, 11.2500, 14.0556, 22.4092); + + stats.AddValue(25); + CheckStats(stats, 19, 1, 100, 9, 6, 4, 16.5000, 12.5000, 14.6316, 21.9221); + + stats.AddValue(17); + CheckStats(stats, 20, 1, 100, 10, 7, 4.5000, 17.2500, 12.7500, 14.7500, 21.3440); + + stats.AddValue(18); + CheckStats(stats, 21, 1, 100, 11, 7, 5, 18, 13, 14.9048, 20.8156); + + stats.AddValue(10); + CheckStats(stats, 22, 1, 100, 10.5000, 7, 5.5000, 17.7500, 12.2500, 14.6818, 20.3409); + + stats.AddValue(22); + CheckStats(stats, 23, 1, 100, 11, 7, 6, 18, 12, 15, 19.9317); + } + +} // Y_UNIT_TEST_SUITE(KqpBenches) + +} // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp index 0eb19a994f03..28a36fc53715 100644 --- a/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_topology_ut.cpp @@ -598,104 +598,6 @@ Y_UNIT_TEST_SUITE(KqpJoinTopology) { RunBenches(ctx, config, args); } - void CheckStats(TRunningStatistics& stats, ui32 n, double min, double max, double median, double mad, double q1, double q3, double iqr, double mean, double stdev) { - const double TOLERANCE = 0.0001; - - UNIT_ASSERT_EQUAL(stats.GetN(), n); - UNIT_ASSERT_DOUBLES_EQUAL(stats.GetMin(), min, TOLERANCE); - UNIT_ASSERT_DOUBLES_EQUAL(stats.GetMax(), max, TOLERANCE); - UNIT_ASSERT_DOUBLES_EQUAL(stats.GetMedian(), median, TOLERANCE); - UNIT_ASSERT_DOUBLES_EQUAL(stats.CalculateMAD(), mad, TOLERANCE); - - auto statistics = stats.GetStatistics(); - UNIT_ASSERT_EQUAL(statistics.GetN(), n); - UNIT_ASSERT_DOUBLES_EQUAL(statistics.GetMin(), min, TOLERANCE); - UNIT_ASSERT_DOUBLES_EQUAL(statistics.GetMax(), max, TOLERANCE); - - auto report = statistics.ComputeStatistics(); - UNIT_ASSERT_EQUAL(report.N, n); - UNIT_ASSERT_DOUBLES_EQUAL(report.Min, min, TOLERANCE); - UNIT_ASSERT_DOUBLES_EQUAL(report.Max, max, TOLERANCE); - UNIT_ASSERT_DOUBLES_EQUAL(report.Median, median, TOLERANCE); - UNIT_ASSERT_DOUBLES_EQUAL(report.MAD, mad, TOLERANCE); - UNIT_ASSERT_DOUBLES_EQUAL(report.Q1, q1, TOLERANCE); - UNIT_ASSERT_DOUBLES_EQUAL(report.Q3, q3, TOLERANCE); - UNIT_ASSERT_DOUBLES_EQUAL(report.IQR, iqr, TOLERANCE); - UNIT_ASSERT_DOUBLES_EQUAL(report.Mean, mean, TOLERANCE); - UNIT_ASSERT_DOUBLES_EQUAL(report.Stdev, stdev, TOLERANCE); - } - - Y_UNIT_TEST(TStatistics) { - TRunningStatistics stats; - - stats.AddValue(1); - stats.AddValue(1); - CheckStats(stats, 2, 1, 1, 1, 0, 1, 1, 0, 1, 0); - - stats.AddValue(1); - CheckStats(stats, 3, 1, 1, 1, 0, 1, 1, 0, 1, 0); - - stats.AddValue(1); - CheckStats(stats, 4, 1, 1, 1, 0, 1, 1, 0, 1, 0); - - stats.AddValue(3); - CheckStats(stats, 5, 1, 3, 1, 0, 1, 1, 0, 1.4000, 0.8944); - - stats.AddValue(5); - CheckStats(stats, 6, 1, 5, 1, 0, 1, 2.5000, 1.5000, 2, 1.6733); - - stats.AddValue(7); - CheckStats(stats, 7, 1, 7, 1, 0, 1, 4, 3, 2.7143, 2.4300); - - stats.AddValue(7); - CheckStats(stats, 8, 1, 7, 2, 1, 1, 5.5000, 4.5000, 3.2500, 2.7124); - - stats.AddValue(8); - CheckStats(stats, 9, 1, 8, 3, 2, 1, 7, 6, 3.7778, 2.9907); - - stats.AddValue(100); - CheckStats(stats, 10, 1, 100, 4, 3, 1, 7, 6, 13.4000, 30.5585); - - stats.AddValue(12); - CheckStats(stats, 11, 1, 100, 5, 4, 1, 7.5000, 6.5000, 13.2727, 28.9934); - - stats.AddValue(15); - CheckStats(stats, 12, 1, 100, 6, 5, 1, 9, 8, 13.4167, 27.6486); - - stats.AddValue(11); - CheckStats(stats, 13, 1, 100, 7, 5, 1, 11, 10, 13.2308, 26.4800); - - stats.AddValue(18); - CheckStats(stats, 14, 1, 100, 7, 5.5000, 1.5000, 11.7500, 10.2500, 13.5714, 25.4731); - - stats.AddValue(19); - CheckStats(stats, 15, 1, 100, 7, 6, 2, 13.5000, 11.5000, 13.9333, 24.5865); - - stats.AddValue(14); - CheckStats(stats, 16, 1, 100, 7.5000, 6.5000, 2.5000, 14.2500, 11.7500, 13.9375, 23.7528); - - stats.AddValue(21); - CheckStats(stats, 17, 1, 100, 8, 7, 3, 15, 12, 14.3529, 23.0623); - - stats.AddValue(9); - CheckStats(stats, 18, 1, 100, 8.5000, 6, 3.5000, 14.7500, 11.2500, 14.0556, 22.4092); - - stats.AddValue(25); - CheckStats(stats, 19, 1, 100, 9, 6, 4, 16.5000, 12.5000, 14.6316, 21.9221); - - stats.AddValue(17); - CheckStats(stats, 20, 1, 100, 10, 7, 4.5000, 17.2500, 12.7500, 14.7500, 21.3440); - - stats.AddValue(18); - CheckStats(stats, 21, 1, 100, 11, 7, 5, 18, 13, 14.9048, 20.8156); - - stats.AddValue(10); - CheckStats(stats, 22, 1, 100, 10.5000, 7, 5.5000, 17.7500, 12.2500, 14.6818, 20.3409); - - stats.AddValue(22); - CheckStats(stats, 23, 1, 100, 11, 7, 6, 18, 12, 15, 19.9317); - } - } // Y_UNIT_TEST_SUITE(KqpJoinTopology) } // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/ut/join/ya.make b/ydb/core/kqp/ut/join/ya.make index aaa937d6d5a8..50e8c2ea9070 100644 --- a/ydb/core/kqp/ut/join/ya.make +++ b/ydb/core/kqp/ut/join/ya.make @@ -26,6 +26,7 @@ SRCS( kqp_join_order_ut.cpp kqp_join_topology_generator.cpp kqp_join_topology_ut.cpp + kqp_benches_ut.cpp ) PEERDIR(