Add base fulltext helpers

ydb/core/base/fulltext.cpp

@@ -0,0 +1,222 @@
+#include "fulltext.h"
+#include <regex>
+
+namespace NKikimr::NFulltext {
+
+namespace {
+
+    Ydb::Table::FulltextIndexSettings::Layout ParseLayout(const TString& layout, TString& error) {
+        if (layout == "flat")
+            return Ydb::Table::FulltextIndexSettings::FLAT;
+        else {
+            error = TStringBuilder() << "Invalid layout: " << layout;
+            return Ydb::Table::FulltextIndexSettings::LAYOUT_UNSPECIFIED;
+        }
+    };
+
+    Ydb::Table::FulltextIndexSettings::Tokenizer ParseTokenizer(const TString& tokenizer, TString& error) {
+        if (tokenizer == "whitespace")
+            return Ydb::Table::FulltextIndexSettings::WHITESPACE;
+        else if (tokenizer == "standard")
+            return Ydb::Table::FulltextIndexSettings::STANDARD;
+        else if (tokenizer == "keyword")
+            return Ydb::Table::FulltextIndexSettings::KEYWORD;
+        else {
+            error = TStringBuilder() << "Invalid tokenizer: " << tokenizer;
+            return Ydb::Table::FulltextIndexSettings::TOKENIZER_UNSPECIFIED;
+        }
+    };
+
+    i32 ParseInt32(const TString& name, const TString& value, TString& error) {
+        i32 result = 0;
+        if (!TryFromString(value, result) || result < 0) { // proto int32 fields with [(Ydb.value) = ">= 0"] annotation
+            error = TStringBuilder() << "Invalid " << name << ": " << value;
+        }
+        return result;
+    }
+
+    bool ParseBool(const TString& name, const TString& value, TString& error) {
+        bool result = false;
+        if (!TryFromString(value, result)) {
+            error = TStringBuilder() << "Invalid " << name << ": " << value;
+        }
+        return result;
+    }
+
+    // Note: written by llm, can be optimized a lot later
+    TVector<TString> Tokenize(const TString& text, const Ydb::Table::FulltextIndexSettings::Tokenizer& tokenizer) {
+        TVector<TString> tokens;
+        switch (tokenizer) {
+            case Ydb::Table::FulltextIndexSettings::WHITESPACE: {
+                std::istringstream stream(text);
+                TString token;
+                while (stream >> token) {
+                    tokens.push_back(token);
+                }
+                break;
+            }
+            case Ydb::Table::FulltextIndexSettings::STANDARD: {
+                std::regex word_regex(R"(\b\w+\b)"); // match alphanumeric words
+                std::sregex_iterator it(text.begin(), text.end(), word_regex);
+                std::sregex_iterator end;
+                while (it != end) {
+                    tokens.push_back(it->str());
+                    ++it;
+                }
+                break;
+            }
+            case Ydb::Table::FulltextIndexSettings::KEYWORD:
+                tokens.push_back(text);
+                break;
+            default:
+                Y_ENSURE(TStringBuilder() << "Invalid tokenizer: " << static_cast<int>(tokenizer));
+        }
+
+        return tokens;
+    }
+
+    bool ValidateSettings(const Ydb::Table::FulltextIndexSettings::Analyzers& settings, TString& error) {
+        if (!settings.has_tokenizer() || settings.tokenizer() == Ydb::Table::FulltextIndexSettings::TOKENIZER_UNSPECIFIED) {
+            error = "tokenizer should be set";
+            return false;
+        }
+
+        if (settings.has_language()) {
+            error = "Unsupported language setting";
+            return false;
+        }
+
+        if (settings.use_filter_stopwords()) {
+            error = "Unsupported use_filter_stopwords setting";
+            return false;
+        }
+
+        if (settings.use_filter_ngram()) {
+            error = "Unsupported use_filter_ngram setting";
+            return false;
+        }
+        if (settings.use_filter_edge_ngram()) {
+            error = "Unsupported use_filter_edge_ngram setting";
+            return false;
+        }
+        if (settings.has_filter_ngram_min_length()) {
+            error = "Unsupported filter_ngram_min_length setting";
+            return false;
+        }
+        if (settings.has_filter_ngram_max_length()) {
+            error = "Unsupported filter_ngram_max_length setting";
+            return false;
+        }
+
+        if (settings.use_filter_length()) {
+            error = "Unsupported use_filter_length setting";
+            return false;
+        }
+        if (settings.has_filter_length_min()) {
+            error = "Unsupported filter_length_min setting";
+            return false;
+        }
+        if (settings.has_filter_length_max()) {
+            error = "Unsupported filter_length_max setting";
+            return false;
+        }
+
+        return true;
+    }
+}
+
+TVector<TString> Analyze(const TString& text, const Ydb::Table::FulltextIndexSettings::Analyzers& settings) {
+    TVector<TString> tokens = Tokenize(text, settings.tokenizer());
+
+    if (settings.use_filter_lowercase()) {
+        for (auto& token : tokens) {
+            token.to_lower();
+        }
+    }
+
+    return tokens;
+}
+
+bool ValidateSettings(const Ydb::Table::FulltextIndexSettings& settings, TString& error) {
+    if (!settings.has_layout() || settings.layout() == Ydb::Table::FulltextIndexSettings::LAYOUT_UNSPECIFIED) {
+        error = "layout should be set";
+        return false;
+    }
+
+    if (settings.columns().size() != 1) {
+        error = TStringBuilder() << "fulltext index should have single column settings"
+            << " but have " << settings.columns().size() << " of them";
+        return false;
+    }
+
+    for (auto column : settings.columns()) {
+        if (!column.has_column()) {
+            error = "column should be set";
+            return false;
+        }
+        if (!column.has_analyzers()) {
+            error = "column analyzers should be set";
+            return false;
+        }
+        if (!ValidateSettings(column.analyzers(), error)) {
+            return false;
+        }
+    }
+
+    error = "";
+    return true;
+}
+
+Ydb::Table::FulltextIndexSettings FillSettings(const TString& column, const TVector<std::pair<TString, TString>>& settings, TString& error) {
+    Ydb::Table::FulltextIndexSettings result;
+    Ydb::Table::FulltextIndexSettings::Analyzers resultAnalyzers;
+
+    for (const auto& [name, value] : settings) {
+        if (name == "layout") {
+            result.set_layout(ParseLayout(value, error));
+        } else if (name == "tokenizer") {
+            resultAnalyzers.set_tokenizer(ParseTokenizer(value, error));
+        } else if (name == "language") {
+            resultAnalyzers.set_language(value);
+        } else if (name == "use_filter_lowercase") {
+            resultAnalyzers.set_use_filter_lowercase(ParseBool(name, value, error));
+        } else if (name == "use_filter_stopwords") {
+            resultAnalyzers.set_use_filter_stopwords(ParseBool(name, value, error));
+        } else if (name == "use_filter_ngram") {
+            resultAnalyzers.set_use_filter_ngram(ParseBool(name, value, error));
+        } else if (name == "use_filter_edge_ngram") {
+            resultAnalyzers.set_use_filter_edge_ngram(ParseBool(name, value, error));
+        } else if (name == "filter_ngram_min_length") {
+            resultAnalyzers.set_filter_ngram_min_length(ParseInt32(name, value, error));
+        } else if (name == "filter_ngram_max_length") {
+            resultAnalyzers.set_filter_ngram_max_length(ParseInt32(name, value, error));
+        } else if (name == "use_filter_length") {
+            resultAnalyzers.set_use_filter_length(ParseBool(name, value, error));
+        } else if (name == "filter_length_min") {
+            resultAnalyzers.set_filter_length_min(ParseInt32(name, value, error));
+        } else if (name == "filter_length_max") {
+            resultAnalyzers.set_filter_length_max(ParseInt32(name, value, error));
+        } else {
+            error = TStringBuilder() << "Unknown index setting: " << name;
+            return result;
+        }
+
+        if (error) {
+            return result;
+        }
+    }
+
+    {
+        // only single-columned index is supported for now
+        auto columnAnalyzers = result.add_columns();
+        columnAnalyzers->set_column(column);
+        columnAnalyzers->mutable_analyzers()->CopyFrom(resultAnalyzers);
+    }
+
+    ValidateSettings(result, error);
+
+    return result;
+}
+
+
+}

ydb/core/base/fulltext.h

@@ -0,0 +1,14 @@
+#pragma once
+
+#include "defs.h"
+
+#include <ydb/public/api/protos/ydb_table.pb.h>
+
+namespace NKikimr::NFulltext {
+
+TVector<TString> Analyze(const TString& text, const Ydb::Table::FulltextIndexSettings::Analyzers& settings);
+
+bool ValidateSettings(const Ydb::Table::FulltextIndexSettings& settings, TString& error);
+Ydb::Table::FulltextIndexSettings FillSettings(const TString& column, const TVector<std::pair<TString, TString>>& values, TString& error);
+
+}

ydb/core/base/kmeans_clusters.cpp

@@ -483,6 +483,7 @@ bool ValidateSettings(const Ydb::Table::KMeansTreeSettings& settings, TString& e
         return false;
     }
 
+    error = "";
     return true;
 }
 
@@ -516,6 +517,7 @@ bool ValidateSettings(const Ydb::Table::VectorIndexSettings& settings, TString&
         return false;
     }
 
+    error = "";
     return true;
 }
 

ydb/core/base/ut/fulltext_ut.cpp

@@ -0,0 +1,105 @@
+#include "fulltext.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+namespace NKikimr::NFulltext {
+
+Y_UNIT_TEST_SUITE(NFulltext) {
+
+    Y_UNIT_TEST(ValidateSettings) {
+        Ydb::Table::FulltextIndexSettings settings;
+        TString error;
+
+        UNIT_ASSERT(!ValidateSettings(settings, error));
+        UNIT_ASSERT_VALUES_EQUAL(error, "layout should be set");
+        settings.set_layout(Ydb::Table::FulltextIndexSettings::FLAT);
+
+        UNIT_ASSERT(!ValidateSettings(settings, error));
+        UNIT_ASSERT_VALUES_EQUAL(error, "fulltext index should have single column settings but have 0 of them");
+        auto columnSettings = settings.add_columns();
+
+        UNIT_ASSERT(!ValidateSettings(settings, error));
+        UNIT_ASSERT_VALUES_EQUAL(error, "column should be set");
+        columnSettings->set_column("text");
+
+        UNIT_ASSERT(!ValidateSettings(settings, error));
+        UNIT_ASSERT_VALUES_EQUAL(error, "column analyzers should be set");
+        auto columnAnalyzers = columnSettings->mutable_analyzers();
+
+        UNIT_ASSERT(!ValidateSettings(settings, error));
+        UNIT_ASSERT_VALUES_EQUAL(error, "tokenizer should be set");
+        columnAnalyzers->set_tokenizer(Ydb::Table::FulltextIndexSettings::STANDARD);
+
+        UNIT_ASSERT_C(ValidateSettings(settings, error), error);
+        UNIT_ASSERT_VALUES_EQUAL(error, "");
+    }
+
+    Y_UNIT_TEST(FillSettings) {
+        TVector<std::pair<TString, TString>> list{
+            {"layout", "flat"},
+            {"tokenizer", "standard"},
+            {"use_filter_lowercase", "true"}
+        };
+
+        TString error;
+        auto settings = FillSettings("text", list, error);
+        UNIT_ASSERT_VALUES_EQUAL(error, "");
+
+        UNIT_ASSERT_EQUAL(settings.layout(), Ydb::Table::FulltextIndexSettings::FLAT);
+        UNIT_ASSERT_VALUES_EQUAL(settings.columns().size(), 1);
+        UNIT_ASSERT_VALUES_EQUAL(settings.columns().at(0).column(), "text");
+        UNIT_ASSERT_EQUAL(settings.columns().at(0).analyzers().tokenizer(), Ydb::Table::FulltextIndexSettings::STANDARD);
+        UNIT_ASSERT_VALUES_EQUAL(settings.columns().at(0).analyzers().use_filter_lowercase(), true);
+    }
+
+    Y_UNIT_TEST(FillSettingsInvalid) {
+        {
+            TVector<std::pair<TString, TString>> list{
+                {"asdf", "qwer"}
+            };
+            TString error;
+            auto settings = FillSettings("text", list, error);
+            UNIT_ASSERT_VALUES_EQUAL(error, "Unknown index setting: asdf");
+        }
+
+        {
+            TVector<std::pair<TString, TString>> list{
+                {"layout", "flat"},
+                {"tokenizer", "standard"},
+                {"use_filter_lowercase", "asdf"}
+            };
+            TString error;
+            auto settings = FillSettings("text", list, error);
+            UNIT_ASSERT_VALUES_EQUAL(error, "Invalid use_filter_lowercase: asdf");
+        }
+
+        {
+            TVector<std::pair<TString, TString>> list{
+                {"layout", "flat"},
+            };
+            TString error;
+            auto settings = FillSettings("text", list, error);
+            UNIT_ASSERT_VALUES_EQUAL(error, "tokenizer should be set");
+        }
+    }
+
+    Y_UNIT_TEST(Analyze) {
+        Ydb::Table::FulltextIndexSettings::Analyzers analyzers;
+        TString text = "apple WaLLet  spaced-dog";
+
+        analyzers.set_tokenizer(Ydb::Table::FulltextIndexSettings::WHITESPACE);
+        UNIT_ASSERT_VALUES_EQUAL(Analyze(text, analyzers), (TVector<TString>{"apple", "WaLLet", "spaced-dog"}));
+
+        analyzers.set_tokenizer(Ydb::Table::FulltextIndexSettings::STANDARD);
+        UNIT_ASSERT_VALUES_EQUAL(Analyze(text, analyzers), (TVector<TString>{"apple", "WaLLet", "spaced", "dog"}));
+
+        analyzers.set_tokenizer(Ydb::Table::FulltextIndexSettings::KEYWORD);
+        UNIT_ASSERT_VALUES_EQUAL(Analyze(text, analyzers), (TVector<TString>{text}));
+
+        analyzers.set_tokenizer(Ydb::Table::FulltextIndexSettings::WHITESPACE);
+        analyzers.set_use_filter_lowercase(true);
+        UNIT_ASSERT_VALUES_EQUAL(Analyze(text, analyzers), (TVector<TString>{"apple", "wallet", "spaced-dog"}));
+    }
+}
+
+}

ydb/core/base/ut/ya.make

@@ -9,13 +9,14 @@ PEERDIR(
 )
 
 SRCS(
-    path_ut.cpp
     blobstorage_grouptype_ut.cpp
+    fulltext_ut.cpp
     localdb_ut.cpp
     logoblob_ut.cpp
     memory_stats_ut.cpp
-    statestorage_ut.cpp
+    path_ut.cpp
     statestorage_guardian_impl_ut.cpp
+    statestorage_ut.cpp
     table_index_ut.cpp
 )
 

ydb/core/base/ya.make

@@ -28,6 +28,8 @@ SRCS(
     feature_flags.h
     feature_flags_service.cpp
     feature_flags_service.h
+    fulltext.cpp
+    fulltext.h
     group_stat.cpp
     group_stat.h
     hive.h

ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp

@@ -152,6 +152,10 @@ TVector<ISubOperation::TPtr> CreateIndexedTable(TOperationId nextId, const TTxTr
                 if (!context.SS->EnableFulltextIndex) {
                     return {CreateReject(nextId, NKikimrScheme::EStatus::StatusPreconditionFailed, "Fulltext index support is disabled")};
                 }
+                TString msg;
+                if (!NKikimr::NFulltext::ValidateSettings(indexDescription.GetFulltextIndexDescription().GetSettings(), msg)) {
+                    return {CreateReject(nextId, NKikimrScheme::EStatus::StatusInvalidParameter, msg)};
+                }
                 break;
             }
             default: