fix bug

.github/workflows/pull_request.yml

@@ -181,17 +181,17 @@ jobs:
             case ${{ matrix.os }} in
             centos7)
               # normal cluster
-              make CONTAINERIZED=true up
+              make CONTAINERIZED=true ENABLE_FT_INDEX=true ES_ADDRESS='"elasticsearch":9200' up
               ;;
             ubuntu2004)
               # ssl cluster
-              make CONTAINERIZED=true ENABLE_SSL=true CA_SIGNED=true up
+              make CONTAINERIZED=true ENABLE_FT_INDEX=true ES_ADDRESS='"elasticsearch":9200' ENABLE_SSL=true CA_SIGNED=true up
               ;;
             esac
             ;;
           clang-*)
             # graph ssl only cluster
-            make CONTAINERIZED=true ENABLE_SSL=false ENABLE_GRAPH_SSL=true up
+            make CONTAINERIZED=true ENABLE_FT_INDEX=true ES_ADDRESS='"elasticsearch":9200' ENABLE_SSL=false ENABLE_GRAPH_SSL=true up
             ;;
           esac
         working-directory: tests/
@@ -203,7 +203,7 @@ jobs:
         timeout-minutes: 15
       - name: TCK
         run: |
-          make RM_DIR=false DEBUG=false J=${{ steps.cmake.outputs.j }} tck
+          make RM_DIR=false DEBUG=false ENABLE_FT_INDEX=true ES_ADDRESS='"elasticsearch":9200' J=${{ steps.cmake.outputs.j }} tck
         working-directory: tests/
         timeout-minutes: 60
       - name: LDBC

src/kvstore/listener/elasticsearch/ESListener.cpp

@@ -20,6 +20,11 @@ void ESListener::init() {
     LOG(FATAL) << "vid length error";
   }
   vIdLen_ = vRet.value();
+  auto vidTypeRet = schemaMan_->getSpaceVidType(spaceId_);
+  if (!vidTypeRet.ok()) {
+    LOG(FATAL) << "vid type error:" << vidTypeRet.status().message();
+  }
+  isIntVid_ = vidTypeRet.value() == nebula::cpp2::PropertyType::INT64;
 
   auto cRet = schemaMan_->getServiceClients(meta::cpp2::ExternalServiceType::ELASTICSEARCH);
   if (!cRet.ok() || cRet.value().empty()) {
@@ -105,6 +110,7 @@ void ESListener::pickTagAndEdgeData(BatchLogType type,
       }
       std::string indexName = index.first;
       std::string vid = NebulaKeyUtils::getVertexId(vIdLen_, key).toString();
+      vid = truncateVid(vid);
       std::string text = std::move(v).getStr();
       callback(type, indexName, vid, "", "", 0, text);
     }
@@ -137,6 +143,8 @@ void ESListener::pickTagAndEdgeData(BatchLogType type,
       std::string dst = NebulaKeyUtils::getDstId(vIdLen_, key).toString();
       int64_t rank = NebulaKeyUtils::getRank(vIdLen_, key);
       std::string text = std::move(v).getStr();
+      src = truncateVid(src);
+      dst = truncateVid(dst);
       callback(type, indexName, "", src, dst, rank, text);
     }
   }
@@ -356,5 +364,12 @@ std::tuple<nebula::cpp2::ErrorCode, int64_t, int64_t> ESListener::commitSnapshot
   return {nebula::cpp2::ErrorCode::SUCCEEDED, count, size};
 }
 
+std::string ESListener::truncateVid(const std::string& vid) {
+  if (!isIntVid_) {
+    return folly::rtrim(folly::StringPiece(vid), [](char c) { return c == '\0'; }).toString();
+  }
+  return vid;
+}
+
 }  // namespace kvstore
 }  // namespace nebula

src/kvstore/listener/elasticsearch/ESListener.h

@@ -117,10 +117,13 @@ class ESListener : public Listener {
                           const std::string& key,
                           const std::string& value,
                           const PickFunc& func);
+
+  std::string truncateVid(const std::string& vid);
   std::unique_ptr<std::string> lastApplyLogFile_{nullptr};
   std::unique_ptr<std::string> spaceName_{nullptr};
   ::nebula::plugin::ESAdapter esAdapter_;
   int32_t vIdLen_;
+  bool isIntVid_{false};
 };
 
 }  // namespace kvstore

tests/Makefile

@@ -14,7 +14,8 @@ TEST_DIR ?= $(CURR_DIR)
 BUILD_DIR ?= $(CURR_DIR)/../build
 DEBUG ?= true
 J ?= 10
-ENABLE_ES ?= false
+ENABLE_FT_INDEX ?= false
+ES_ADDRESS ?= "locahost:9200"
 ENABLE_SSL ?= false
 ENABLE_GRAPH_SSL ?= false
 ENABLE_META_SSL ?= false
@@ -29,9 +30,10 @@ QUERY_CONCURRENTLY ?= false
 gherkin_fmt = ~/.local/bin/reformat-gherkin
 run_test = PYTHONPATH=$$PYTHONPATH:$(CURR_DIR)/.. $(CURR_DIR)/nebula-test-run.py
 
-ifeq ($(ENABLE_ES),false)
+ifeq ($(ENABLE_FT_INDEX),false)
 	PYTEST_MARKEXPR="not skip and not ft_index"
 else
+	export NEBULA_TEST_ES_ADDRESS=${ES_ADDRESS}
 	PYTEST_MARKEXPR="not skip"
 endif
 

tests/common/nebula_service.py

@@ -594,6 +594,15 @@ def start(self):
         print("add hosts cmd is {}".format(cmd))
         resp = client.execute(cmd)
         assert resp.is_succeeded(), resp.error_msg()
+
+        # sign text search service
+        NEBULA_TEST_ES_ADDRESS = os.environ.get("NEBULA_TEST_ES_ADDRESS")
+        if NEBULA_TEST_ES_ADDRESS is not None:
+            cmd = f"SIGN IN TEXT SERVICE({NEBULA_TEST_ES_ADDRESS});"
+            print("sign text service cmd is {}".format(cmd))
+            resp = client.execute(cmd)
+            assert resp.is_succeeded(), resp.error_msg()
+
         client.release()
 
         # wait nebula start

tests/tck/features/fulltext_index/FulltextIndexQuery1.feature

@@ -12,7 +12,7 @@ Feature: FulltextIndexTest
     And add listeners to space
 
   @ft_index
-  Scenario: fulltext query
+  Scenario: fulltext query1
     When executing query:
       """
       CREATE TAG tag1(prop1 string,prop2 string);
@@ -82,3 +82,27 @@ Feature: FulltextIndexTest
       | src | dst | rank | prop1    |
       | 3   | 4   | 5    | "高性能" |
       | 4   | 5   | 7    | "高吞吐" |
+    When executing query:
+      """
+      LOOKUP ON tag1 where regexp(tag1.prop2,"neBula.*") YIELD id(vertex) as id, tag1.prop1 as prop1
+      """
+    Then the result should be, in any order:
+      | id | prop1 |
+      | 5  | "cba" |
+    When executing query:
+      """
+      LOOKUP ON edge1 where wildcard(edge1.prop1,"高??") YIELD src(edge) as src,dst(edge) as dst,rank(edge) as rank, edge1.prop1 as prop1
+      """
+    Then the result should be, in any order:
+      | src | dst | rank | prop1    |
+      | 3   | 4   | 5    | "高性能" |
+      | 4   | 5   | 7    | "高吞吐" |
+    When executing query:
+      """
+      LOOKUP ON tag1 where fuzzy(edge1.prop2,"nebula") YIELD tag1.prop2 as prop2
+      """
+    Then the result should be, in any order:
+      | prop2    |
+      | "Nebula" |
+      | "neBula" |
+

tests/tck/features/fulltext_index/FulltextIndexQuery2.feature

@@ -0,0 +1,84 @@
+# Copyright (c) 2022 vesoft inc. All rights reserved.
+#
+# This source code is licensed under Apache 2.0 License.
+Feature: FulltextIndexTest
+
+  Background:
+    Given an empty graph
+    And create a space with following options:
+      | partition_num  | 1                |
+      | replica_factor | 1                |
+      | vid_type       | FIXED_STRING(30) |
+    And add listeners to space
+
+  @ft_index
+  Scenario: fulltext query2
+    When executing query:
+      """
+      CREATE TAG tag2(prop1 string,prop2 string);
+      CREATE EDGE edge2(prop1 string);
+      """
+    Then the execution should be successful
+    And wait 3 seconds
+    When executing query:
+      """
+      CREATE FULLTEXT TAG INDEX nebula_index_tag2_prop1 on tag2(prop1);
+      CREATE FULLTEXT TAG INDEX nebula_index_tag2_prop2 on tag2(prop2);
+      CREATE FULLTEXT EDGE INDEX nebula_index_edge2_prop1 on edge2(prop1);
+      """
+    Then the execution should be successful
+    And wait 5 seconds
+    When executing query:
+      """
+      INSERT VERTEX tag2(prop1,prop2) VALUES "1":("abc","nebula graph");
+      INSERT VERTEX tag2(prop1,prop2) VALUES "2":("abcde","nebula-graph");
+      INSERT VERTEX tag2(prop1,prop2) VALUES "3":("bcd","nebula database");
+      INSERT VERTEX tag2(prop1,prop2) VALUES "4":("zyx","Nebula");
+      INSERT VERTEX tag2(prop1,prop2) VALUES "5":("cba","neBula");
+      INSERT VERTEX tag2(prop1,prop2) VALUES "6":("abcxyz","nebula graph");
+      INSERT VERTEX tag2(prop1,prop2) VALUES "7":("xyz","nebula graph");
+      INSERT VERTEX tag2(prop1,prop2) VALUES "8":("123456","nebula graph");
+      """
+    Then the execution should be successful
+    When executing query:
+      """
+      INSERT EDGE edge2(prop1) VALUES "1"->"2"@1:("一个可靠的分布式");
+      INSERT EDGE edge2(prop1) VALUES "2"->"3"@3:("性能高效的图数据库");
+      INSERT EDGE edge2(prop1) VALUES "3"->"4"@5:("高性能");
+      INSERT EDGE edge2(prop1) VALUES "4"->"5"@7:("高吞吐");
+      INSERT EDGE edge2(prop1) VALUES "5"->"6"@9:("低延时");
+      INSERT EDGE edge2(prop1) VALUES "6"->"7"@11:("易扩展");
+      INSERT EDGE edge2(prop1) VALUES "7"->"8"@13:("线性扩缩容");
+      INSERT EDGE edge2(prop1) VALUES "8"->"1"@15:("安全稳定");
+      """
+    Then the execution should be successful
+    And wait 10 seconds
+    When executing query:
+      """
+      LOOKUP ON tag2 where prefix(tag2.prop1,"abc") YIELD id(vertex) as id, tag2.prop1 as prop1, tag2.prop2 as prop2
+      """
+    Then the result should be, in any order:
+      | id | prop1    | prop2          |
+      | "1"  | "abc"    | "nebula graph" |
+      | "2"  | "abcde"  | "nebula-graph" |
+      | "6"  | "abcxyz" | "nebula graph" |
+    When executing query:
+      """
+      LOOKUP ON tag2 where prefix(tag2.prop2,"nebula") YIELD id(vertex) as id, tag2.prop1 as prop1, tag2.prop2 as prop2
+      """
+    Then the result should be, in any order:
+      | id   | prop1    | prop2             |
+      | "1"  | "abc"    | "nebula graph"    |
+      | "2"  | "abcde"  | "nebula-graph"    |
+      | "3"  | "bcd"    | "nebula database" |
+      | "6"  | "abcxyz" | "nebula graph"    |
+      | "7"  | "xyz"    | "nebula graph"    |
+      | "8"  | "123456" | "nebula graph"    |
+    When executing query:
+      """
+      LOOKUP ON edge2 where prefix(edge2.prop1,"高") YIELD src(edge) as src,dst(edge) as dst,rank(edge) as rank, edge2.prop1 as prop1
+      """
+    Then the result should be, in any order:
+      | src   | dst   | rank | prop1    |
+      | "3"   | "4"   | 5    | "高性能" |
+      | "4"   | "5"   | 7    | "高吞吐" |