Unmute CompactionVariants #17929

.github/config/muted_ya.txt

@@ -34,9 +34,6 @@ ydb/core/kqp/ut/cost KqpCost.OlapWriteRow
 ydb/core/kqp/ut/federated_query/large_results KqpScriptExecResults.ExecuteScriptWithLargeFile
 ydb/core/kqp/ut/federated_query/s3 sole chunk chunk
 ydb/core/kqp/ut/indexes KqpMultishardIndex.WriteIntoRenamingSyncIndex
-ydb/core/kqp/ut/olap KqpOlapJson.CompactionVariants
-ydb/core/kqp/ut/olap KqpOlapJson.DuplicationCompactionVariants
-ydb/core/kqp/ut/olap KqpOlapJson.SwitchAccessorCompactionVariants
 ydb/core/kqp/ut/olap KqpOlapWrite.TierDraftsGCWithRestart
 ydb/core/kqp/ut/olap [*/*] chunk chunk
 ydb/core/kqp/ut/query KqpAnalyze.AnalyzeTable+ColumnStore

ydb/core/kqp/ut/olap/combinatory/select.cpp

@@ -16,7 +16,7 @@ TConclusionStatus TSelectCommand::DoExecute(TKikimrRunner& kikimr) {
     const i64 headerApproveStart = controller->GetHeadersApprovedOnSelect().Val();
     const i64 headerNoDataStart = controller->GetHeadersSkippedNoData().Val();
 
-    const auto command = "PRAGMA OptimizeSimpleILIKE; PRAGMA AnsiLIke;" + Command;
+    const auto command = "PRAGMA OptimizeSimpleILIKE; PRAGMA AnsiLike;" + Command;
     Cerr << "EXECUTE: " << command << Endl;
     auto session = kikimr.GetTableClient().CreateSession().GetValueSync().GetSession();
     auto it = kikimr.GetQueryClient().StreamExecuteQuery(command, NYdb::NQuery::TTxControl::BeginTx().CommitTx()).ExtractValueSync();

ydb/core/kqp/ut/olap/combinatory/variator.cpp

@@ -5,107 +5,177 @@
 #include "select.h"
 #include "variator.h"
 
-namespace NKikimr::NKqp {
+namespace NKikimr::NKqp::Variator {
+namespace {
+    using TCommandFactory = std::function<std::shared_ptr<ICommand>()>;
 
-std::shared_ptr<ICommand> TScriptVariator::BuildCommand(const TString command) {
-    std::shared_ptr<ICommand> result;
-    size_t pos = command.find(":");
-    if (pos == TString::npos) {
-        pos = command.find("\n");
-    }
-    if (pos == TString::npos) {
-        pos = command.size();
+    const std::unordered_map<TString, TCommandFactory> CommandFactories = {
+        {"BULK_UPSERT", []() { return std::make_shared<TBulkUpsertCommand>(); }},
+        {"SCHEMA", []() { return std::make_shared<TSchemaCommand>(); }},
+        {"DATA", []() { return std::make_shared<TDataCommand>(); }},
+        {"READ", []() { return std::make_shared<TSelectCommand>(); }},
+        {"WAIT_COMPACTION", []() { return std::make_shared<TWaitCompactionCommand>(); }},
+        {"STOP_COMPACTION", []() { return std::make_shared<TStopCompactionCommand>(); }},
+        {"ONE_COMPACTION", []() { return std::make_shared<TOneCompactionCommand>(); }},
+        {"ONE_ACTUALIZATION", []() { return std::make_shared<TOneActualizationCommand>(); }}
+    };
+
+    std::pair<TString, TString> ParseCommandString(const TString& command) {
+        TString cleanCommand = Strip(command);
+
+        size_t pos = cleanCommand.find(':');
+        if (pos == TString::npos) {
+            return {cleanCommand, TString()};
+        }
+
+        return {Strip(cleanCommand.substr(0, pos)), Strip(cleanCommand.substr(pos + 1))};
     }
-    const TString name = Strip(command.substr(0, pos));
-    if (name == "BULK_UPSERT") {
-        result = std::make_shared<TBulkUpsertCommand>();
-    } else if (name == "SCHEMA") {
-        result = std::make_shared<TSchemaCommand>();
-    } else if (name == "DATA") {
-        result = std::make_shared<TDataCommand>();
-    } else if (name == "READ") {
-        result = std::make_shared<TSelectCommand>();
-    } else if (name == "WAIT_COMPACTION") {
-        result = std::make_shared<TWaitCompactionCommand>();
-    } else if (name == "STOP_COMPACTION") {
-        result = std::make_shared<TStopCompactionCommand>();
-    } else if (name == "ONE_COMPACTION") {
-        result = std::make_shared<TOneCompactionCommand>();
-    } else if (name == "ONE_ACTUALIZATION") {
-        result = std::make_shared<TOneActualizationCommand>();
-    } else {
-        AFL_VERIFY(false)("command", command);
+}
+
+std::shared_ptr<ICommand> BuildCommand(const TString& command) {
+    const auto [commandName, arguments] = ParseCommandString(command);
+
+    auto it = CommandFactories.find(commandName);
+    if (it == CommandFactories.end()) {
+        AFL_VERIFY(false)("Unknown command", command)("commandName", commandName);
         return nullptr;
     }
-    if (pos != command.size()) {
-        const TString arguments = Strip(command.substr(pos + 1));
+
+    auto result = it->second();
+
+    if (!arguments.empty()) {
         result->DeserializeFromString(arguments).Validate();
     }
+
     return result;
 }
 
-void TScriptVariator::BuildScripts(const std::vector<std::vector<std::shared_ptr<ICommand>>>& commands, const ui32 currentLayer,
-    std::vector<std::shared_ptr<ICommand>>& currentScript, std::vector<TScriptExecutor>& scripts) {
+void BuildScripts(
+    const std::vector<std::vector<std::tuple<TString, TString>>>& commands,
+    const ui32 currentLayer,
+    Script& currentScript,
+    std::vector<TString>& currentVariant,
+    std::vector<std::tuple<Script, Varinat>>& scripts
+) {
     if (currentLayer == commands.size()) {
-        scripts.emplace_back(currentScript);
+        scripts.emplace_back(currentScript, JoinSeq(",", currentVariant));
         return;
     }
-    for (auto&& i : commands[currentLayer]) {
-        currentScript.emplace_back(i);
-        BuildScripts(commands, currentLayer + 1, currentScript, scripts);
+
+    for (const auto& [command, variant] : commands[currentLayer]) {
+        if (!variant.empty()) {
+            currentVariant.push_back(variant);
+        }
+
+        currentScript.push_back(command);
+        BuildScripts(commands, currentLayer + 1, currentScript, currentVariant, scripts);
         currentScript.pop_back();
+
+        if (!variant.empty()) {
+            currentVariant.pop_back();
+        }
     }
 }
 
-void TScriptVariator::BuildVariantsImpl(const std::vector<std::vector<TString>>& chunks, const ui32 currentLayer,
-    std::vector<TString>& currentCommand, std::vector<TString>& results) {
+void BuildVariantsImpl(
+    const std::vector<std::vector<TString>>& chunks,
+    const ui32 currentLayer,
+    std::vector<TString>& currentCommand,
+    std::vector<TString>& currentVariant,
+    std::vector<std::tuple<TString, TString>>& results
+) {
     if (currentLayer == chunks.size()) {
-        results.emplace_back(JoinSeq("", currentCommand));
+        results.emplace_back(JoinSeq("", currentCommand), JoinSeq(",", currentVariant));
         return;
     }
-    for (auto&& i : chunks[currentLayer]) {
-        currentCommand.emplace_back(i);
-        BuildVariantsImpl(chunks, currentLayer + 1, currentCommand, results);
+
+    const auto& chunkVariants = chunks[currentLayer];
+    const bool hasMultipleVariants = chunkVariants.size() > 1;
+
+    for (const auto& chunk : chunkVariants) {
+        if (hasMultipleVariants) {
+            currentVariant.push_back(chunk);
+        }
+
+        currentCommand.push_back(chunk);
+        BuildVariantsImpl(chunks, currentLayer + 1, currentCommand, currentVariant, results);
         currentCommand.pop_back();
+
+        if (hasMultipleVariants) {
+            currentVariant.pop_back();
+        }
     }
 }
 
-std::vector<TString> TScriptVariator::BuildVariants(const TString& command) {
+std::vector<std::tuple<TString, TString>> BuildVariants(const TString& command) {
     auto chunks = StringSplitter(command).SplitByString("$$").ToList<TString>();
     std::vector<std::vector<TString>> chunksVariants;
+    chunksVariants.reserve(chunks.size());
+
     for (ui32 i = 0; i < chunks.size(); ++i) {
         if (i % 2 == 0) {
-            chunksVariants.emplace_back(std::vector<TString>({ chunks[i] }));
+            chunksVariants.push_back({chunks[i]});
         } else {
-            chunksVariants.emplace_back(StringSplitter(chunks[i]).SplitBySet("|").ToList<TString>());
+            chunksVariants.push_back(
+                StringSplitter(chunks[i]).SplitBySet("|").ToList<TString>()
+            );
         }
     }
-    std::vector<TString> result;
+
+    std::vector<std::tuple<TString, TString>> result;
     std::vector<TString> currentCommand;
-    BuildVariantsImpl(chunksVariants, 0, currentCommand, result);
+    std::vector<TString> currentVariant;
+
+    BuildVariantsImpl(chunksVariants, 0, currentCommand, currentVariant, result);
     return result;
 }
 
-TScriptVariator::TScriptVariator(const TString& script) {
+std::vector<std::tuple<Script, Varinat>> ScriptVariants(const TString& script) {
+    auto commands = SingleScript(script);
+
+    std::vector<std::vector<std::tuple<TString, TString>>> commandsDescription;
+    commandsDescription.reserve(commands.size());
+
+    for (auto& cmd : commands) {
+        cmd = Strip(cmd);
+        commandsDescription.push_back(BuildVariants(cmd));
+    }
+
+    std::vector<std::tuple<Script, Varinat>> result;
+    Script currentScript;
+    std::vector<TString> currentVariant;
+
+    BuildScripts(commandsDescription, 0, currentScript, currentVariant, result);
+    return result;
+}
+
+Script SingleScript(const TString& script) {
     auto lines = StringSplitter(script).SplitByString("\n").ToList<TString>();
-    lines.erase(std::remove_if(lines.begin(), lines.end(),
-                    [](const TString& l) {
-                        return Strip(l).StartsWith("#");
-                    }),
-        lines.end());
-    auto commands = StringSplitter(JoinSeq("\n", lines)).SplitByString("------").ToList<TString>();
-    std::vector<std::vector<std::shared_ptr<ICommand>>> commandsDescription;
-    for (auto&& i : commands) {
-        auto& cVariants = commandsDescription.emplace_back();
-        i = Strip(i);
-        std::vector<TString> variants = BuildVariants(i);
-        for (auto&& v : variants) {
-            cVariants.emplace_back(BuildCommand(v));
-        }
+
+    lines.erase(
+        std::remove_if(lines.begin(), lines.end(),
+            [](const TString& line) {
+                return Strip(line).StartsWith("#");
+            }),
+        lines.end()
+    );
+
+    return StringSplitter(JoinSeq("\n", lines)).SplitByString("------").ToList<TString>();
+}
+
+TScriptExecutor ToExecutor(const Script& script) {
+    std::vector<std::shared_ptr<ICommand>> result;
+    result.reserve(script.size());
+
+    for (const auto& command : script) {
+        result.push_back(BuildCommand(command));
     }
-    std::vector<TScriptExecutor> scripts;
-    std::vector<std::shared_ptr<ICommand>> scriptCommands;
-    BuildScripts(commandsDescription, 0, scriptCommands, Scripts);
+
+    return TScriptExecutor(result);
+}
+
+TString ToString(const Script& script) {
+    return JoinSeq("\n------\n", script);
 }
 
-}   // namespace NKikimr::NKqp
+}  // namespace NKikimr::NKqp::Variator

ydb/core/kqp/ut/olap/combinatory/variator.h

@@ -1,32 +1,61 @@
 #pragma once
+
 #include "abstract.h"
 #include "executor.h"
 
 #include <ydb/core/kqp/ut/common/kqp_ut_common.h>
-
 #include <ydb/library/conclusion/status.h>
+#include <library/cpp/testing/unittest/registar.h>
+
+namespace NKikimr::NKqp::Variator {
+
+using Script = std::vector<TString>;
+using Varinat = TString;
+
+std::vector<std::tuple<Script, Varinat>> ScriptVariants(const TString& script);
+
+Script SingleScript(const TString& script);
+
+TScriptExecutor ToExecutor(const Script& script);
+
+TString ToString(const Script& script);
+
+#define Y_UNIT_TEST_STRING_VARIATOR(NAME, INPUT_STRING)                                    \
+    struct TTestCase##NAME : public TCurrentTestCase {                                     \
+        TString Script;                                                                    \
+        TString Name;                                                                      \
+                                                                                           \
+        TTestCase##NAME(TString script, TString name)                                      \
+            : Script(std::move(script))                                                    \
+            , Name(TString(#NAME) + "[" + name + "]")                                      \
+        {                                                                                  \
+            Name_ = Name.c_str();                                                          \
+            ForceFork_ = false;                                                             \
+        }                                                                                  \
+                                                                                           \
+        static THolder<NUnitTest::TBaseTestCase> Create(TString script, TString name) {    \
+            return ::MakeHolder<TTestCase##NAME>(std::move(script), std::move(name));      \
+        }                                                                                  \
+                                                                                           \
+        void Execute_(NUnitTest::TTestContext& ut_context Y_DECLARE_UNUSED) override {     \
+            ExecuteImpl(Script);                                                           \
+        }                                                                                  \
+                                                                                           \
+        void ExecuteImpl(const TString& script);                                           \
+    };                                                                                     \
+                                                                                           \
+    struct TTestRegistration##NAME {                                                       \
+        TTestRegistration##NAME() {                                                        \
+            for (const auto& [script, variant] : Variator::ScriptVariants(INPUT_STRING)) { \
+                TCurrentTest::AddTest([script, variant] {                                  \
+                    return TTestCase##NAME::Create(Variator::ToString(script), variant);   \
+                });                                                                        \
+            }                                                                              \
+        }                                                                                  \
+    };                                                                                     \
+                                                                                           \
+    static const TTestRegistration##NAME testRegistration##NAME;                           \
+                                                                                           \
+    void TTestCase##NAME::ExecuteImpl(const TString& __SCRIPT_CONTENT)
+} // namespace NKikimr::NKqp::Variator
 
-namespace NKikimr::NKqp {
-
-class TScriptVariator {
-private:
-    std::vector<TScriptExecutor> Scripts;
-    std::shared_ptr<ICommand> BuildCommand(const TString command);
-    void BuildScripts(const std::vector<std::vector<std::shared_ptr<ICommand>>>& commands, const ui32 currentLayer,
-        std::vector<std::shared_ptr<ICommand>>& currentScript, std::vector<TScriptExecutor>& scripts);
-
-    void BuildVariantsImpl(const std::vector<std::vector<TString>>& chunks, const ui32 currentLayer, std::vector<TString>& currentCommand,
-        std::vector<TString>& results);
-    std::vector<TString> BuildVariants(const TString& command);
-
-public:
-    TScriptVariator(const TString& script);
-
-    void Execute() {
-        for (auto&& i : Scripts) {
-            i.Execute();
-        }
-    }
-};
-
-}   // namespace NKikimr::NKqp