From ec5d2ac801f1bf18176d4082d833291737591e25 Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Tue, 27 Dec 2022 19:02:20 +0800 Subject: [PATCH] 1. fulltext index add ddl test case (#5122) 2. fix ESListener crash when there is no elasticsearch --- src/interface/common.thrift | 2 + .../listener/elasticsearch/ESListener.cpp | 48 +++++--- .../listener/elasticsearch/ESListener.h | 4 +- .../processors/index/FTIndexProcessor.cpp | 6 +- .../fulltext_index/FultextIndexDDL.feature | 109 ++++++++++++++++++ 5 files changed, 147 insertions(+), 22 deletions(-) create mode 100644 tests/tck/features/fulltext_index/FultextIndexDDL.feature diff --git a/src/interface/common.thrift b/src/interface/common.thrift index 150a52f459d..61c925df258 100644 --- a/src/interface/common.thrift +++ b/src/interface/common.thrift @@ -421,6 +421,8 @@ enum ErrorCode { E_QUERY_NOT_FOUND = -2073, // Query not found E_AGENT_HB_FAILUE = -2074, // Failed to receive heartbeat from agent + E_ACCESS_ES_FAILURE = -2080, // Failed to access elasticsearch + // 3xxx for storaged E_CONSENSUS_ERROR = -3001, // Consensus cannot be reached during an election E_KEY_HAS_EXISTS = -3002, // Key already exists diff --git a/src/kvstore/listener/elasticsearch/ESListener.cpp b/src/kvstore/listener/elasticsearch/ESListener.cpp index f9119fd01a7..cdc4fe5205d 100644 --- a/src/kvstore/listener/elasticsearch/ESListener.cpp +++ b/src/kvstore/listener/elasticsearch/ESListener.cpp @@ -26,22 +26,6 @@ void ESListener::init() { } isIntVid_ = vidTypeRet.value() == nebula::cpp2::PropertyType::INT64; - auto cRet = schemaMan_->getServiceClients(meta::cpp2::ExternalServiceType::ELASTICSEARCH); - if (!cRet.ok() || cRet.value().empty()) { - LOG(FATAL) << "elasticsearch clients error"; - } - std::vector esClients; - for (const auto& c : cRet.value()) { - auto host = c.host; - std::string user, password; - if (c.user_ref().has_value()) { - user = *c.user_ref(); - password = *c.pwd_ref(); - } - std::string protocol = c.conn_type_ref().has_value() ? *c.get_conn_type() : "http"; - esClients.emplace_back(HttpClient::instance(), protocol, host.toRawString(), user, password); - } - esAdapter_.setClients(std::move(esClients)); auto sRet = schemaMan_->toGraphSpaceName(spaceId_); if (!sRet.ok()) { LOG(FATAL) << "space name error"; @@ -70,7 +54,13 @@ bool ESListener::apply(const BatchHolder& batch) { pickTagAndEdgeData(std::get<0>(log), std::get<1>(log), std::get<2>(log), callback); } if (!bulk.empty()) { - auto status = esAdapter_.bulk(bulk); + auto esAdapterRes = getESAdapter(); + if (!esAdapterRes.ok()) { + LOG(ERROR) << esAdapterRes.status(); + return false; + } + auto esAdapter = std::move(esAdapterRes).value(); + auto status = esAdapter.bulk(bulk); if (!status.ok()) { LOG(ERROR) << status; return false; @@ -371,5 +361,29 @@ std::string ESListener::truncateVid(const std::string& vid) { return vid; } +StatusOr<::nebula::plugin::ESAdapter> ESListener::getESAdapter() { + auto cRet = schemaMan_->getServiceClients(meta::cpp2::ExternalServiceType::ELASTICSEARCH); + if (!cRet.ok()) { + LOG(ERROR) << cRet.status().message(); + return cRet.status(); + } + if (cRet.value().empty()) { + LOG(ERROR) << "There is no elasticsearch service"; + return ::nebula::Status::Error("There is no elasticsearch service"); + } + std::vector esClients; + for (const auto& c : cRet.value()) { + auto host = c.host; + std::string user, password; + if (c.user_ref().has_value()) { + user = *c.user_ref(); + password = *c.pwd_ref(); + } + std::string protocol = c.conn_type_ref().has_value() ? *c.get_conn_type() : "http"; + esClients.emplace_back(HttpClient::instance(), protocol, host.toRawString(), user, password); + } + return ::nebula::plugin::ESAdapter(std::move(esClients)); +} + } // namespace kvstore } // namespace nebula diff --git a/src/kvstore/listener/elasticsearch/ESListener.h b/src/kvstore/listener/elasticsearch/ESListener.h index 33073f4eb96..b46cc58f830 100644 --- a/src/kvstore/listener/elasticsearch/ESListener.h +++ b/src/kvstore/listener/elasticsearch/ESListener.h @@ -119,9 +119,11 @@ class ESListener : public Listener { const PickFunc& func); std::string truncateVid(const std::string& vid); + + StatusOr<::nebula::plugin::ESAdapter> getESAdapter(); + std::unique_ptr lastApplyLogFile_{nullptr}; std::unique_ptr spaceName_{nullptr}; - ::nebula::plugin::ESAdapter esAdapter_; int32_t vIdLen_; bool isIntVid_{false}; }; diff --git a/src/meta/processors/index/FTIndexProcessor.cpp b/src/meta/processors/index/FTIndexProcessor.cpp index ad263788aa9..9771187ecfa 100644 --- a/src/meta/processors/index/FTIndexProcessor.cpp +++ b/src/meta/processors/index/FTIndexProcessor.cpp @@ -139,9 +139,8 @@ void CreateFTIndexProcessor::process(const cpp2::CreateFTIndexReq& req) { plugin::ESAdapter esAdapter(std::move(esClients)); auto createIndexresult = esAdapter.createIndex(name); if (!createIndexresult.ok()) { - // TODO(hs.zhang): fix error code LOG(ERROR) << createIndexresult.message(); - handleErrorCode(nebula::cpp2::ErrorCode::E_UNKNOWN); + handleErrorCode(nebula::cpp2::ErrorCode::E_ACCESS_ES_FAILURE); onFinished(); return; } @@ -209,9 +208,8 @@ void DropFTIndexProcessor::process(const cpp2::DropFTIndexReq& req) { plugin::ESAdapter esAdapter(std::move(esClients)); auto dropIndexresult = esAdapter.dropIndex(req.get_fulltext_index_name()); if (!dropIndexresult.ok()) { - // TODO(hs.zhang): fix error code LOG(ERROR) << dropIndexresult.message(); - handleErrorCode(nebula::cpp2::ErrorCode::E_UNKNOWN); + handleErrorCode(nebula::cpp2::ErrorCode::E_ACCESS_ES_FAILURE); onFinished(); return; } diff --git a/tests/tck/features/fulltext_index/FultextIndexDDL.feature b/tests/tck/features/fulltext_index/FultextIndexDDL.feature new file mode 100644 index 00000000000..d21d68f753b --- /dev/null +++ b/tests/tck/features/fulltext_index/FultextIndexDDL.feature @@ -0,0 +1,109 @@ +# Copyright (c) 2022 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. +Feature: FulltextIndexTest + + Background: + Given an empty graph + And create a space with following options: + | partition_num | 1 | + | replica_factor | 1 | + | vid_type | FIXED_STRING(30) | + And add listeners to space + + @ft_index + Scenario: fulltext ddl + When executing query: + """ + CREATE TAG ddl_tag(prop1 string,prop2 fixed_string(20),prop3 int); + CREATE EDGE ddl_edge(prop1 string,prop2 float); + """ + Then the execution should be successful + And wait 3 seconds + When executing query: + """ + CREATE FULLTEXT TAG INDEX nebula_index_ddl_tag_prop1 on ddl_tag(prop1); + """ + Then the execution should be successful + When executing query: + """ + CREATE FULLTEXT TAG INDEX nebula_index_ddl_tag_prop2 on ddl_tag(prop2); + """ + Then the execution should be successful + When executing query: + """ + CREATE FULLTEXT TAG INDEX nebula_index_ddl_tag_prop3 on ddl_tag(prop3); + """ + Then a ExecutionError should be raised at runtime: Unsupported! + When executing query: + """ + SHOW FULLTEXT INDEXES; + """ + Then the result should be, in any order: + | Name | Schema Type | Schema Name | Fields | + | "nebula_index_ddl_tag_prop1" | "Tag" | "ddl_tag" | "prop1" | + | "nebula_index_ddl_tag_prop2" | "Tag" | "ddl_tag" | "prop2" | + When executing query: + """ + DROP FULLTEXT INDEX nebula_index_ddl_tag_prop1; + """ + Then the execution should be successful + When executing query: + """ + DROP FULLTEXT INDEX nebula_index_ddl_tag_prop2; + """ + Then the execution should be successful + When executing query: + """ + SHOW FULLTEXT INDEXES; + """ + Then the result should be, in any order: + | Name | Schema Type | Schema Name | Fields | + When executing query: + """ + CREATE FULLTEXT TAG INDEX nebula_index_ddl_tag_prop1 on ddl_tag(prop2); + """ + Then the execution should be successful + When executing query: + """ + SHOW FULLTEXT INDEXES; + """ + Then the result should be, in any order: + | Name | Schema Type | Schema Name | Fields | + | "nebula_index_ddl_tag_prop1" | "Tag" | "ddl_tag" | "prop2" | + When executing query: + """ + DROP TAG ddl_tag; + """ + Then a ExecutionError should be raised at runtime: Related index exists, please drop index first + When executing query: + """ + ALTER TAG ddl_tag DROP (prop2); + """ + Then a ExecutionError should be raised at runtime: Related fulltext index exists, please drop it first + When executing query: + """ + ALTER TAG ddl_tag DROP (prop1); + """ + Then the execution should be successful + When executing query: + """ + ALTER TAG ddl_tag ADD (prop1 string); + """ + Then the execution should be successful + When executing query: + """ + ALTER TAG ddl_tag CHANGE (prop2 string); + """ + Then a ExecutionError should be raised at runtime: Related fulltext index exists, please drop it first + When executing query: + """ + DROP FULLTEXT INDEX nebula_index_ddl_tag_prop1; + """ + Then the execution should be successful + When executing query: + """ + DROP TAG ddl_tag; + DROP EDGE ddl_edge; + """ + Then the execution should be successful