diff --git a/conf/nebula-graphd.conf.default b/conf/nebula-graphd.conf.default index 8ec1e6129da..919f94d7f5c 100644 --- a/conf/nebula-graphd.conf.default +++ b/conf/nebula-graphd.conf.default @@ -90,3 +90,9 @@ ########## experimental feature ########## # if use experimental features --enable_experimental_feature=false + +# if use toss feature, only work if enable_experimental_feature is true +--enable_toss=false + +# if use balance data feature, only work if enable_experimental_feature is true +--enable_data_balance=true diff --git a/conf/nebula-graphd.conf.production b/conf/nebula-graphd.conf.production index 538231bd679..b79c9d4dba4 100644 --- a/conf/nebula-graphd.conf.production +++ b/conf/nebula-graphd.conf.production @@ -90,6 +90,12 @@ # if use experimental features --enable_experimental_feature=false +# if use toss feature, only work if enable_experimental_feature is true +--enable_toss=false + +# if use balance data feature, only work if enable_experimental_feature is true +--enable_data_balance=true + ########## session ########## # Maximum number of sessions that can be created per IP and per user --max_sessions_per_ip_per_user=300 diff --git a/conf/nebula-standalone.conf.default b/conf/nebula-standalone.conf.default index a735ab3cf3a..8e7b9ad7702 100644 --- a/conf/nebula-standalone.conf.default +++ b/conf/nebula-standalone.conf.default @@ -92,6 +92,12 @@ # if use experimental features --enable_experimental_feature=false +# if use toss feature, only work if enable_experimental_feature is true +--enable_toss=false + +# if use balance data feature, only work if enable_experimental_feature is true +--enable_data_balance=true + ######### Raft ######### # Raft election timeout --raft_heartbeat_interval_secs=30 diff --git a/src/common/hdfs/HdfsCommandHelper.cpp b/src/common/hdfs/HdfsCommandHelper.cpp index 14e6b7f98b8..117d3ffb6af 100644 --- a/src/common/hdfs/HdfsCommandHelper.cpp +++ b/src/common/hdfs/HdfsCommandHelper.cpp @@ -24,6 +24,9 @@ Status HdfsCommandHelper::ls(const std::string& hdfsHost, auto result = proc.wait(); if (!result.exited()) { return Status::Error("Failed to ls hdfs"); + } else if (result.exitStatus() != 0) { + LOG(INFO) << "Failed to ls: " << result.str(); + return Status::Error("Failed to ls hdfs, errno: %d", result.exitStatus()); } else { return Status::OK(); } @@ -47,6 +50,9 @@ Status HdfsCommandHelper::copyToLocal(const std::string& hdfsHost, auto result = proc.wait(); if (!result.exited()) { return Status::Error("Failed to download from hdfs"); + } else if (result.exitStatus() != 0) { + LOG(INFO) << "Failed to download: " << result.str(); + return Status::Error("Failed to download from hdfs, errno: %d", result.exitStatus()); } else { return Status::OK(); } diff --git a/src/graph/executor/mutate/DeleteExecutor.cpp b/src/graph/executor/mutate/DeleteExecutor.cpp index b3df2670e68..4377b95b2b0 100644 --- a/src/graph/executor/mutate/DeleteExecutor.cpp +++ b/src/graph/executor/mutate/DeleteExecutor.cpp @@ -200,7 +200,7 @@ folly::Future DeleteEdgesExecutor::deleteEdges() { auto plan = qctx()->plan(); StorageClient::CommonRequestParam param( spaceId, qctx()->rctx()->session()->id(), plan->id(), plan->isProfileEnabled()); - param.useExperimentalFeature = FLAGS_enable_experimental_feature; + param.useExperimentalFeature = FLAGS_enable_experimental_feature && FLAGS_enable_toss; return qctx() ->getStorageClient() ->deleteEdges(param, std::move(edgeKeys)) diff --git a/src/graph/executor/mutate/InsertExecutor.cpp b/src/graph/executor/mutate/InsertExecutor.cpp index 2b082c35d97..2375d5e35c5 100644 --- a/src/graph/executor/mutate/InsertExecutor.cpp +++ b/src/graph/executor/mutate/InsertExecutor.cpp @@ -54,7 +54,7 @@ folly::Future InsertEdgesExecutor::insertEdges() { auto plan = qctx()->plan(); StorageClient::CommonRequestParam param( ieNode->getSpace(), qctx()->rctx()->session()->id(), plan->id(), plan->isProfileEnabled()); - param.useExperimentalFeature = FLAGS_enable_experimental_feature; + param.useExperimentalFeature = FLAGS_enable_experimental_feature && FLAGS_enable_toss; return qctx() ->getStorageClient() ->addEdges(param, diff --git a/src/graph/executor/mutate/UpdateExecutor.cpp b/src/graph/executor/mutate/UpdateExecutor.cpp index 63e774b22d0..ac331b10375 100644 --- a/src/graph/executor/mutate/UpdateExecutor.cpp +++ b/src/graph/executor/mutate/UpdateExecutor.cpp @@ -98,7 +98,7 @@ folly::Future UpdateEdgeExecutor::execute() { auto plan = qctx()->plan(); StorageClient::CommonRequestParam param( ueNode->getSpaceId(), qctx()->rctx()->session()->id(), plan->id(), plan->isProfileEnabled()); - param.useExperimentalFeature = FLAGS_enable_experimental_feature; + param.useExperimentalFeature = FLAGS_enable_experimental_feature && FLAGS_enable_toss; return qctx() ->getStorageClient() ->updateEdge(param, diff --git a/src/graph/service/GraphFlags.cpp b/src/graph/service/GraphFlags.cpp index 5ea5fbc1ad3..07a65c4b02c 100644 --- a/src/graph/service/GraphFlags.cpp +++ b/src/graph/service/GraphFlags.cpp @@ -70,6 +70,8 @@ DEFINE_bool(disable_octal_escape_char, " in next version to ensure compatibility with cypher."); DEFINE_bool(enable_experimental_feature, false, "Whether to enable experimental feature"); +DEFINE_bool(enable_toss, false, "Whether to enable toss feature"); +DEFINE_bool(enable_data_balance, true, "Whether to enable data balance feature"); DEFINE_int32(num_rows_to_check_memory, 1024, "number rows to check memory"); DEFINE_int32(max_sessions_per_ip_per_user, @@ -103,3 +105,5 @@ DEFINE_uint32( gc_worker_size, 0, "Background garbage clean workers, default number is 0 which means using hardware core size."); + +DEFINE_bool(graph_use_vertex_key, false, "whether allow insert or query the vertex key"); diff --git a/src/graph/service/GraphFlags.h b/src/graph/service/GraphFlags.h index f2c7ecf9c75..6cb54849762 100644 --- a/src/graph/service/GraphFlags.h +++ b/src/graph/service/GraphFlags.h @@ -54,6 +54,8 @@ DECLARE_bool(optimize_appendvertice); DECLARE_int64(max_allowed_connections); DECLARE_bool(enable_experimental_feature); +DECLARE_bool(enable_toss); +DECLARE_bool(enable_data_balance); DECLARE_bool(enable_client_white_list); DECLARE_string(client_white_list); @@ -65,4 +67,7 @@ DECLARE_int32(max_job_size); DECLARE_bool(enable_async_gc); DECLARE_uint32(gc_worker_size); + +DECLARE_bool(graph_use_vertex_key); + #endif // GRAPH_GRAPHFLAGS_H_ diff --git a/src/graph/validator/AdminJobValidator.cpp b/src/graph/validator/AdminJobValidator.cpp index c1b8ae2fad6..5e7d6b635d5 100644 --- a/src/graph/validator/AdminJobValidator.cpp +++ b/src/graph/validator/AdminJobValidator.cpp @@ -6,6 +6,7 @@ #include "graph/validator/AdminJobValidator.h" #include "graph/planner/plan/Admin.h" +#include "graph/service/GraphFlags.h" namespace nebula { namespace graph { @@ -13,7 +14,7 @@ namespace graph { Status AdminJobValidator::validateImpl() { if (sentence_->getJobType() == meta::cpp2::JobType::DATA_BALANCE || sentence_->getJobType() == meta::cpp2::JobType::ZONE_BALANCE) { - if (!FLAGS_enable_experimental_feature) { + if (!(FLAGS_enable_experimental_feature && FLAGS_enable_data_balance)) { return Status::SemanticError("Data balance not support"); } } diff --git a/src/graph/validator/MutateValidator.cpp b/src/graph/validator/MutateValidator.cpp index 41f1f6fb14f..ce16006280d 100644 --- a/src/graph/validator/MutateValidator.cpp +++ b/src/graph/validator/MutateValidator.cpp @@ -46,6 +46,9 @@ Status InsertVerticesValidator::check() { } auto tagItems = sentence->tagItems(); + if (!FLAGS_graph_use_vertex_key && tagItems.empty()) { + return Status::SemanticError("Insert vertex is forbidden, please speicify the tag"); + } schemas_.reserve(tagItems.size()); @@ -206,7 +209,8 @@ Status InsertEdgesValidator::check() { // Check validity of vertices data. // Check edge key type, check properties value, fill to NewEdge structure. Status InsertEdgesValidator::prepareEdges() { - auto size = FLAGS_enable_experimental_feature ? rows_.size() : rows_.size() * 2; + auto size = + FLAGS_enable_experimental_feature && FLAGS_enable_toss ? rows_.size() : rows_.size() * 2; edges_.reserve(size); size_t fieldNum = schema_->getNumFields(); @@ -291,7 +295,7 @@ Status InsertEdgesValidator::prepareEdges() { edge.key_ref() = key; edge.props_ref() = std::move(entirePropValues); edges_.emplace_back(edge); - if (!FLAGS_enable_experimental_feature) { + if (!(FLAGS_enable_experimental_feature && FLAGS_enable_toss)) { // inbound key.src_ref() = dstId; key.dst_ref() = srcId; @@ -826,7 +830,7 @@ Status UpdateEdgeValidator::toPlan() { {}, condition_, {}); - if (FLAGS_enable_experimental_feature) { + if ((FLAGS_enable_experimental_feature && FLAGS_enable_toss)) { root_ = outNode; tail_ = root_; } else { diff --git a/src/graph/validator/test/YieldValidatorTest.cpp b/src/graph/validator/test/YieldValidatorTest.cpp index fa4d47ef13e..4d4b3aa1926 100644 --- a/src/graph/validator/test/YieldValidatorTest.cpp +++ b/src/graph/validator/test/YieldValidatorTest.cpp @@ -162,15 +162,11 @@ TEST_F(YieldValidatorTest, TypeCastTest) { } { std::string query = "YIELD (int)\"123abc\""; - auto result = checkResult(query); - EXPECT_EQ(std::string(result.message()), - "SemanticError: `(INT)\"123abc\"' is not a valid expression "); + EXPECT_TRUE(checkResult(query)); } { std::string query = "YIELD (int)\"abc123\""; - auto result = checkResult(query); - EXPECT_EQ(std::string(result.message()), - "SemanticError: `(INT)\"abc123\"' is not a valid expression "); + EXPECT_TRUE(checkResult(query)); } { std::string query = "YIELD (doublE)\"123\""; @@ -182,9 +178,7 @@ TEST_F(YieldValidatorTest, TypeCastTest) { } { std::string query = "YIELD (doublE)\".a123\""; - auto result = checkResult(query); - EXPECT_EQ(std::string(result.message()), - "SemanticError: `(FLOAT)\".a123\"' is not a valid expression "); + EXPECT_TRUE(checkResult(query)); } { std::string query = "YIELD (STRING)1.23"; @@ -200,15 +194,15 @@ TEST_F(YieldValidatorTest, TypeCastTest) { } { std::string query = "YIELD (BOOL)123"; - EXPECT_FALSE(checkResult(query, expected_)); + EXPECT_TRUE(checkResult(query, expected_)); } { std::string query = "YIELD (BOOL)0"; - EXPECT_FALSE(checkResult(query, expected_)); + EXPECT_TRUE(checkResult(query, expected_)); } { std::string query = "YIELD (BOOL)\"12\""; - EXPECT_FALSE(checkResult(query, expected_)); + EXPECT_TRUE(checkResult(query, expected_)); } { std::string query = "YIELD (MAP)(\"12\")"; diff --git a/src/graph/visitor/DeduceTypeVisitor.cpp b/src/graph/visitor/DeduceTypeVisitor.cpp index 16fccff4e78..bf8ab94b3ca 100644 --- a/src/graph/visitor/DeduceTypeVisitor.cpp +++ b/src/graph/visitor/DeduceTypeVisitor.cpp @@ -249,13 +249,7 @@ void DeduceTypeVisitor::visit(TypeCastingExpression *expr) { status_ = Status::SemanticError(out.str()); return; } - QueryExpressionContext ctx(nullptr); - auto val = expr->eval(ctx(nullptr)); - if (val.isNull()) { - status_ = Status::SemanticError("`%s' is not a valid expression ", expr->toString().c_str()); - return; - } - type_ = val.type(); + type_ = expr->type(); status_ = Status::OK(); } @@ -486,7 +480,21 @@ void DeduceTypeVisitor::visit(FunctionCallExpression *expr) { void DeduceTypeVisitor::visit(AggregateExpression *expr) { expr->arg()->accept(this); if (!ok()) return; - type_ = Value::Type::__EMPTY__; + auto func = expr->name(); + std::transform(func.begin(), func.end(), func.begin(), ::toupper); + if ("COUNT" == func) { + type_ = Value::Type::INT; + } else if ("COLLECT" == func) { + type_ = Value::Type::LIST; + } else if ("COLLECT_SET" == func) { + type_ = Value::Type::SET; + } else if ("AVG" == func || "SUM" == func) { + type_ = Value::Type::FLOAT; + } else if ("MAX" == func || "MIN" == func) { + // Keep same with arg's type + } else { + type_ = Value::Type::__EMPTY__; + } } void DeduceTypeVisitor::visit(UUIDExpression *) { diff --git a/src/storage/admin/RebuildIndexTask.cpp b/src/storage/admin/RebuildIndexTask.cpp index 98e0545cb7b..e4cbad43fdf 100644 --- a/src/storage/admin/RebuildIndexTask.cpp +++ b/src/storage/admin/RebuildIndexTask.cpp @@ -17,6 +17,17 @@ const int32_t kReserveNum = 1024 * 4; bool RebuildIndexTask::check() { return env_->kvstore_ != nullptr; } +void RebuildIndexTask::finish(nebula::cpp2::ErrorCode rc) { + if (changedSpaceGuard_) { + auto space = *ctx_.parameters_.space_id_ref(); + for (auto it = env_->rebuildIndexGuard_->begin(); it != env_->rebuildIndexGuard_->end(); ++it) { + if (std::get<0>(it->first) == space) { + env_->rebuildIndexGuard_->insert_or_assign(it->first, IndexState::FINISHED); + } + } + } + AdminTask::finish(rc); +} RebuildIndexTask::RebuildIndexTask(StorageEnv* env, TaskContext&& ctx) : AdminTask(env, std::move(ctx)) { @@ -71,6 +82,7 @@ ErrorOr> RebuildIndexTask::ge for (const auto& part : parts) { env_->rebuildIndexGuard_->insert_or_assign(std::make_tuple(space_, part), IndexState::STARTING); TaskFunction task = std::bind(&RebuildIndexTask::invoke, this, space_, part, items); + changedSpaceGuard_ = true; tasks.emplace_back(std::move(task)); } return tasks; diff --git a/src/storage/admin/RebuildIndexTask.h b/src/storage/admin/RebuildIndexTask.h index 2f3eac072b9..7e8b0fd1a58 100644 --- a/src/storage/admin/RebuildIndexTask.h +++ b/src/storage/admin/RebuildIndexTask.h @@ -23,6 +23,8 @@ using IndexItems = std::vector>; */ class RebuildIndexTask : public AdminTask { public: + using AdminTask::finish; + RebuildIndexTask(StorageEnv* env, TaskContext&& ctx); ~RebuildIndexTask() { @@ -31,6 +33,8 @@ class RebuildIndexTask : public AdminTask { bool check() override; + void finish(nebula::cpp2::ErrorCode rc) override; + /** * @brief Generate subtasks for rebuilding index. * @@ -71,6 +75,7 @@ class RebuildIndexTask : public AdminTask { protected: GraphSpaceID space_; + bool changedSpaceGuard_{false}; }; } // namespace storage diff --git a/src/storage/exec/GetPropNode.h b/src/storage/exec/GetPropNode.h index 4f2bf6a495f..9b0be49cbb3 100644 --- a/src/storage/exec/GetPropNode.h +++ b/src/storage/exec/GetPropNode.h @@ -21,7 +21,8 @@ class GetTagPropNode : public QueryNode { std::vector tagNodes, nebula::DataSet* resultDataSet, Expression* filter, - std::size_t limit) + std::size_t limit, + TagContext* tagContext) : context_(context), tagNodes_(std::move(tagNodes)), resultDataSet_(resultDataSet), @@ -29,7 +30,8 @@ class GetTagPropNode : public QueryNode { ? nullptr : new StorageExpressionContext(context->vIdLen(), context->isIntId())), filter_(filter), - limit_(limit) { + limit_(limit), + tagContext_(tagContext) { name_ = "GetTagPropNode"; } @@ -69,7 +71,36 @@ class GetTagPropNode : public QueryNode { } else if (!iter->valid()) { return nebula::cpp2::ErrorCode::SUCCEEDED; } - // if has any tag, will emplace a row with vId + + bool hasValidTag = false; + for (; iter->valid(); iter->next()) { + // check if tag schema exists + auto key = iter->key(); + auto tagId = NebulaKeyUtils::getTagId(context_->vIdLen(), key); + auto schemaIter = tagContext_->schemas_.find(tagId); + if (schemaIter == tagContext_->schemas_.end()) { + continue; + } + // check if ttl expired + auto schemas = &(schemaIter->second); + RowReaderWrapper reader; + reader.reset(*schemas, iter->val()); + if (!reader) { + continue; + } + auto ttl = QueryUtils::getTagTTLInfo(tagContext_, tagId); + if (ttl.has_value() && + CommonUtils::checkDataExpiredForTTL( + schemas->back().get(), reader.get(), ttl.value().first, ttl.value().second)) { + continue; + } + hasValidTag = true; + break; + } + if (!hasValidTag) { + return nebula::cpp2::ErrorCode::SUCCEEDED; + } + // if has any valid tag, will emplace a row with vId } } @@ -131,6 +162,7 @@ class GetTagPropNode : public QueryNode { std::unique_ptr expCtx_{nullptr}; Expression* filter_{nullptr}; const std::size_t limit_{std::numeric_limits::max()}; + TagContext* tagContext_; }; class GetEdgePropNode : public QueryNode { diff --git a/src/storage/query/GetPropProcessor.cpp b/src/storage/query/GetPropProcessor.cpp index 21231ef8217..1c1303a3131 100644 --- a/src/storage/query/GetPropProcessor.cpp +++ b/src/storage/query/GetPropProcessor.cpp @@ -208,7 +208,7 @@ StoragePlan GetPropProcessor::buildTagPlan(RuntimeContext* context, plan.addNode(std::move(tag)); } auto output = std::make_unique( - context, tags, result, filter_ == nullptr ? nullptr : filter_->clone(), limit_); + context, tags, result, filter_ == nullptr ? nullptr : filter_->clone(), limit_, &tagContext_); for (auto* tag : tags) { output->addDependency(tag); } diff --git a/tests/tck/features/delete/DeleteTag.IntVid.feature b/tests/tck/features/delete/DeleteTag.IntVid.feature index 70eadfc8626..1bf23a2d956 100644 --- a/tests/tck/features/delete/DeleteTag.IntVid.feature +++ b/tests/tck/features/delete/DeleteTag.IntVid.feature @@ -161,7 +161,6 @@ Feature: Delete int vid of tag | id | Then drop the used space - @wtf Scenario: delete int vid multiple vertex one tag Given an empty graph And load "nba_int_vid" csv data to a new space diff --git a/tests/tck/features/go/GO.feature b/tests/tck/features/go/GO.feature index 7374d9bcdaa..7b719d3f51d 100644 --- a/tests/tck/features/go/GO.feature +++ b/tests/tck/features/go/GO.feature @@ -149,6 +149,16 @@ Feature: Go Sentence | "Spurs" | | "Hornets" | | "Trail Blazers" | + When executing query: + """ + GO FROM "Tim Duncan" OVER like WHERE like._rank<10 YIELD like._src AS src, like._dst AS dst, like._rank AS rank + | YIELD $-.src AS src, $-.dst AS dst, max($-.rank) AS maxRank + | FETCH PROP ON like $-.src -> $-.dst@$-.maxRank YIELD edge AS e + """ + Then the result should be, in any order, with relax comparison: + | e | + | [:like "Tim Duncan"->"Tony Parker" @0 {likeness: 95}] | + | [:like "Tim Duncan"->"Manu Ginobili" @0 {likeness: 95}] | Scenario: In expression When executing query: diff --git a/tests/tck/features/insert/insertVertexOnly.feature b/tests/tck/features/insert/insertVertexOnly.feature index 29ebff743d9..9419e29e54b 100644 --- a/tests/tck/features/insert/insertVertexOnly.feature +++ b/tests/tck/features/insert/insertVertexOnly.feature @@ -19,6 +19,10 @@ Feature: insert vertex without tag When executing query and retrying it on failure every 6 seconds for 3 times: """ INSERT VERTEX VALUES 1:(),2:(),3:(); + """ + Then a SemanticError should be raised at runtime: Insert vertex is forbidden, please speicify the tag + When executing query: + """ INSERT EDGE e() VALUES 1->2:(),2->3:(); """ Then the execution should be successful diff --git a/tests/tck/features/match/Base.feature b/tests/tck/features/match/Base.feature index 1d0c826ee7d..27348d21ef9 100644 --- a/tests/tck/features/match/Base.feature +++ b/tests/tck/features/match/Base.feature @@ -718,3 +718,46 @@ Feature: Basic match Then the result should be, in any order: | v | | ("Boris Diaw" :player{age: 36, name: "Boris Diaw"}) | + + Scenario: Match with id when all tag is dropped, ent-#1420 + Given an empty graph + And create a space with following options: + | partition_num | 1 | + | replica_factor | 1 | + | vid_type | FIXED_STRING(20) | + And having executed: + """ + CREATE TAG IF NOT EXISTS player(name string, age int); + CREATE TAG IF NOT EXISTS team(name string); + CREATE TAG INDEX IF NOT EXISTS player_index_1 on player(name(10), age); + """ + And wait 5 seconds + When try to execute query: + """ + INSERT VERTEX player() VALUES "v2":(); + INSERT VERTEX player(name, age) VALUES "v3":("v3", 18); + UPSERT VERTEX ON player "v4" SET name = "v4", age = 18; + """ + Then the execution should be successful + When executing query: + """ + MATCH (v) WHERE id(v) in ["v1", "v2", "v3", "v4"] return id(v) limit 10; + """ + Then the result should be, in any order, with relax comparison: + | id(v) | + | "v2" | + | "v3" | + | "v4" | + When try to execute query: + """ + DROP TAG INDEX player_index_1; + DROP TAG player; + """ + Then the execution should be successful + And wait 5 seconds + When executing query: + """ + MATCH (v) WHERE id(v) in ["v1", "v2", "v3", "v4"] return id(v) limit 10; + """ + Then the result should be, in any order, with relax comparison: + | id(v) | diff --git a/tests/tck/features/ttl/TTL.feature b/tests/tck/features/ttl/TTL.feature index f3f74c6c618..a5aaec23196 100644 --- a/tests/tck/features/ttl/TTL.feature +++ b/tests/tck/features/ttl/TTL.feature @@ -393,22 +393,19 @@ Feature: TTLTest FETCH PROP ON person "1" YIELD vertex as node; """ Then the result should be, in any order, with relax comparison: - | node | - | ("1") | + | node | When executing query: """ FETCH PROP ON person "1" YIELD person.id as id """ Then the result should be, in any order: - | id | - | EMPTY | + | id | When executing query: """ FETCH PROP ON * "1" YIELD person.id, career.id """ Then the result should be, in any order: | person.id | career.id | - | EMPTY | EMPTY | When executing query: """ FETCH PROP ON person "2" YIELD person.id @@ -491,6 +488,5 @@ Feature: TTLTest FETCH PROP ON person "1" YIELD person.age as age; """ Then the result should be, in any order: - | age | - | EMPTY | + | age | And drop the used space