From db865a09f56f57afdbbe2adb5bd5f4e6aaadad47 Mon Sep 17 00:00:00 2001 From: fickludd Date: Thu, 28 Jun 2018 08:53:44 +0200 Subject: [PATCH] Large refactor of execution result PLAN DESCRIPTIONS: Plan descriptions are now generated directly from the logical plan, instead of being based on the physical operators. This makes the plan descriptions for all runtimes be the same, appart from any added arguments. To enable profiling, runtimes now produce a `QueryProfile` if the plan is executed with profiling turned on. This query profile is then handed to the plan description builder to enrich the plan description. INTERNAL EXECUTION RESULT: The InternalExecutionResult has been cleaned of test-only code - that functionality now lives in RewindableExecutionResult. Also, things have changed so that StandardInternadExecutionResult now is it's on class responsible for execution meta data and policies around materializing results early. ClosingExecutionResult is responsible for all closing of the execution results due to end-of-rows or exceptions. Runtimes implement a much slimmer RuntimeResult instead of InternalExecutionResult. This only holds the primary functionality related to getting results and profile information. Notably no runtime is any longer doing any custom closing, results materialization, or value conversion. --- .../cypher/ExecutionEngineTestSupport.scala | 8 +- .../neo4j/cypher/ExecutionResultTest.scala | 6 +- .../org/neo4j/cypher/MergeConcurrencyIT.scala | 2 +- .../RewindableExecutionResultTest.scala | 61 ---- .../neo4j/cypher/RootPlanAcceptanceTest.scala | 6 +- .../cypher/TxCountsTrackingTestSupport.scala | 4 +- .../compiler/v3_5/Notifications.scala | 6 + ...cedureCallOrSchemaCommandPlanBuilder.scala | 8 +- .../org/neo4j/cypher/internal/Compiler.scala | 11 +- .../cypher/internal/MasterCompiler.scala | 75 +++-- .../internal/compatibility/BasePlanner.scala | 2 +- .../ClosingExecutionResult.scala | 212 +++++------- .../compatibility/CypherCurrentCompiler.scala | 95 ++++-- .../compatibility/CypherPlanner.scala | 12 +- .../compatibility/InterpretedRuntime.scala | 32 +- .../ProcedureCallOrSchemaCommandRuntime.scala | 79 +++-- .../compatibility/v2_3/Cypher23Compiler.scala | 14 +- .../v2_3/ExecutionResultWrapper.scala | 26 +- .../compatibility/v3_1/Cypher31Compiler.scala | 28 +- .../v3_1/ExecutionResultWrapper.scala | 33 +- .../compatibility/v3_1/typeConversions.scala | 37 +-- .../compatibility/v3_4/Cypher34Planner.scala | 6 +- .../compatibility/v3_5/Cypher35Planner.scala | 8 +- .../ExceptionTranslatingQueryContext.scala | 5 +- .../v3_5/runtime/ExecutionResultDumper.scala | 143 -------- .../v3_5/runtime/ExplainExecutionResult.scala | 27 +- ...etable.scala => IteratorBasedResult.scala} | 14 +- .../v3_5/runtime/PipeExecutionResult.scala | 118 ++----- .../v3_5/runtime/ResultIterator.scala | 115 ------- ...DefaultExecutionResultBuilderFactory.scala | 86 ++--- .../runtime/executionplan/ExecutionPlan.scala | 7 +- .../ExecutionResultBuilder.scala | 17 +- .../v3_5/runtime/executionplan/Provider.java | 5 + .../StandardInternalExecutionResult.scala | 304 +++++++++--------- .../runtime/executionplan/formatOutput.scala | 6 +- .../procs/ProcedureCallExecutionPlan.scala | 80 +---- ...scala => ProcedureCallRuntimeResult.scala} | 124 +++---- .../procs/PureSideEffectExecutionPlan.scala | 70 ---- ...ureSideEffectInternalExecutionResult.scala | 61 ---- .../procs/SchemaWriteExecutionPlan.scala | 50 +++ .../procs/SchemaWriteRuntimeResult.scala | 53 +++ .../runtime/helpers/InternalWrapping.scala | 6 + .../helpers/RuntimeTextValueConverter.scala | 102 +++++- .../InterpretedProfileInformation.scala | 53 +++ .../profiler/PlanDescriptionBuilder.scala | 59 ++++ .../v3_5/runtime/profiler/Profiler.scala | 69 ++-- .../neo4j/cypher/QueryPlanTestSupport.scala | 8 +- .../cypher/QueryStatisticsTestSupport.scala | 125 +++---- .../internal/RewindableExecutionResult.scala | 151 +++++---- .../v3_5/runtime/ClosingIteratorTest.scala | 121 ------- .../runtime/PipeExecutionPlanBuilderIT.scala | 2 +- .../ExecutionWorkflowBuilderTest.scala | 101 ------ .../StandardInternalExecutionResultTest.scala | 146 +++++++++ .../ProcedureCallExecutionPlanTest.scala | 30 +- .../v3_5/runtime/profiler/ProfilerTest.scala | 126 +++----- .../interpreted/DelegatingQueryContext.scala | 5 +- .../TransactionBoundQueryContext.scala | 4 +- .../pipes/ConstraintOperationPipe.scala | 45 --- .../pipes/IndexOperationPipe.scala | 55 ---- .../interpreted/pipes/PipeDecorator.scala | 5 - .../pipes/PipeExecutionBuilderContext.scala | 4 +- .../interpreted/QueryContextAdaptation.scala | 3 - .../pipes/ProcedureCallPipeTest.scala | 2 - .../neo4j/cypher/result/OperatorProfile.java | 92 ++++++ .../org/neo4j/cypher/result/QueryProfile.java | 40 +++ .../neo4j/cypher/result/RuntimeResult.java | 47 +++ .../runtime/InternalExecutionResult.scala | 36 ++- .../internal/runtime/QueryContext.scala | 6 +- .../internal/runtime/QueryStatistics.scala | 2 +- .../LogicalPlan2PlanDescription.scala | 39 ++- .../BuiltInProcedureAcceptanceTest.scala | 11 +- .../acceptance/CypherComparisonSupport.scala | 109 +++++-- .../DumpToStringAcceptanceTest.scala | 6 +- .../EagerizationAcceptanceTest.scala | 12 +- .../acceptance/ExecutionEngineTest.scala | 19 +- .../acceptance/ExplainAcceptanceTest.scala | 6 +- .../acceptance/LoadCsvAcceptanceTest.scala | 8 +- .../LoadCsvWithQuotesAcceptanceTest.scala | 2 +- .../acceptance/MatchAcceptanceTest.scala | 10 +- ...ionsBackedByCountStoreAcceptanceTest.scala | 5 +- .../MergeIntoPlanningAcceptanceTest.scala | 6 +- .../NotificationAcceptanceTest.scala | 4 +- .../ParameterValuesAcceptanceTest.scala | 4 +- ...pressionImplementationAcceptanceTest.scala | 4 +- .../PeriodicCommitAcceptanceTest.scala | 1 - .../acceptance/ProfilerAcceptanceTest.scala | 21 +- .../acceptance/SameQueryStressTest.scala | 6 +- .../SerializationAcceptanceTest.scala | 50 +-- .../cypher/acceptance/SetAcceptanceTest.scala | 4 +- .../ShortestPathAcceptanceTest.scala | 2 - .../ShortestPathLongerAcceptanceTest.scala | 6 +- .../ShortestPathSameNodeAcceptanceTest.scala | 2 +- .../SpatialFunctionsAcceptanceTest.scala | 2 +- .../SpatialIndexResultsAcceptanceTest.scala | 5 +- .../acceptance/TemporalAcceptanceTest.scala | 6 +- .../TriadicSelectionAcceptanceTest.scala | 2 +- .../UnsupportedFeaturesAcceptanceTest.scala | 4 +- .../UpdateReportingAcceptanceTest.scala | 2 +- .../codegen/profiling/ProfilingTracer.java | 43 +-- .../GeneratedQueryExecution.java | 3 - .../cypher/internal/CompiledRuntime.scala | 58 +--- .../neo4j/cypher/internal/MorselRuntime.scala | 106 +++--- .../cypher/internal/SlottedRuntime.scala | 6 +- .../compiled/CompiledExecutionResult.scala | 49 ++- .../compiled/ExecutionPlanBuilder.scala | 74 ----- .../runtime/compiled/RunnablePlan.scala | 42 +++ .../compiled/codegen/CodeGenerator.scala | 25 +- .../cypher/internal/spi/codegen/Fields.scala | 1 - .../spi/codegen/GeneratedQueryStructure.scala | 7 +- .../internal/spi/codegen/Templates.scala | 7 +- .../neo4j/cypher/RootPlanAcceptanceTest.scala | 10 +- .../profiling/ProfilingTracerTest.scala | 4 +- .../compiled_runtime/TypeConversionTest.scala | 4 +- .../CypherReductionSupport.scala | 16 +- .../CypherReductionSupportTest.scala | 8 +- .../codegen/CompiledExecutionResultTest.scala | 230 ------------- .../GeneratedMethodStructureTest.scala | 3 +- .../ir/BuildProbeTableInstructionsTest.scala | 2 +- .../spi/codegen/ir/CodeGenSugar.scala | 68 ++-- .../codegen/ir/CompiledProfilingTest.scala | 4 +- .../runtime/vectorized/Dispatcher.scala | 12 +- ...SlottedExecutionResultBuilderFactory.scala | 11 +- .../slotted/SlottedPipeBuilderTest.scala | 2 +- 123 files changed, 1947 insertions(+), 2707 deletions(-) delete mode 100644 community/community-it/cypher-it/src/test/scala/org/neo4j/cypher/RewindableExecutionResultTest.scala delete mode 100644 community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/ExecutionResultDumper.scala rename community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/{executionplan/Completable.scala => IteratorBasedResult.scala} (68%) delete mode 100644 community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/ResultIterator.scala rename community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/{ProcedureExecutionResult.scala => ProcedureCallRuntimeResult.scala} (59%) delete mode 100644 community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/PureSideEffectExecutionPlan.scala delete mode 100644 community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/PureSideEffectInternalExecutionResult.scala create mode 100644 community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/SchemaWriteExecutionPlan.scala create mode 100644 community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/SchemaWriteRuntimeResult.scala create mode 100644 community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/profiler/InterpretedProfileInformation.scala create mode 100644 community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/profiler/PlanDescriptionBuilder.scala delete mode 100644 community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/ClosingIteratorTest.scala delete mode 100644 community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/ExecutionWorkflowBuilderTest.scala create mode 100644 community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/StandardInternalExecutionResultTest.scala delete mode 100644 community/cypher/interpreted-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/interpreted/pipes/ConstraintOperationPipe.scala delete mode 100644 community/cypher/interpreted-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/interpreted/pipes/IndexOperationPipe.scala create mode 100644 community/cypher/runtime-util/src/main/java/org/neo4j/cypher/result/OperatorProfile.java create mode 100644 community/cypher/runtime-util/src/main/java/org/neo4j/cypher/result/QueryProfile.java create mode 100644 community/cypher/runtime-util/src/main/java/org/neo4j/cypher/result/RuntimeResult.java delete mode 100644 enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/runtime/compiled/ExecutionPlanBuilder.scala create mode 100644 enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/runtime/compiled/RunnablePlan.scala delete mode 100644 enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/spi/codegen/CompiledExecutionResultTest.scala diff --git a/community/community-it/cypher-it/src/test/scala/org/neo4j/cypher/ExecutionEngineTestSupport.scala b/community/community-it/cypher-it/src/test/scala/org/neo4j/cypher/ExecutionEngineTestSupport.scala index ea33084a16113..185005659827e 100644 --- a/community/community-it/cypher-it/src/test/scala/org/neo4j/cypher/ExecutionEngineTestSupport.scala +++ b/community/community-it/cypher-it/src/test/scala/org/neo4j/cypher/ExecutionEngineTestSupport.scala @@ -134,16 +134,16 @@ trait ExecutionEngineHelper { case x => ValueUtils.of(x) } - def execute(q: String, params: (String, Any)*): InternalExecutionResult = + def execute(q: String, params: (String, Any)*): RewindableExecutionResult = RewindableExecutionResult(eengine.execute(q, asMapValue(params.toMap), graph.transactionalContext(query = q -> params.toMap))) - def execute(q: String, params: Map[String, Any]): InternalExecutionResult = + def execute(q: String, params: Map[String, Any]): RewindableExecutionResult = RewindableExecutionResult(eengine.execute(q, asMapValue(params), graph.transactionalContext(query = q -> params.toMap))) def executeOfficial(q: String, params: (String, Any)*): Result = eengine.execute(q, asMapValue(params.toMap), graph.transactionalContext(query = q -> params.toMap)) - def profile(q: String, params: (String, Any)*): InternalExecutionResult = + def profile(q: String, params: (String, Any)*): RewindableExecutionResult = RewindableExecutionResult(eengine.profile(q, asMapValue(params.toMap), graph.transactionalContext(query = q -> params.toMap))) def executeScalar[T](q: String, params: (String, Any)*): T = { @@ -159,7 +159,7 @@ trait ExecutionEngineHelper { val value: Any = m.head._2 value.asInstanceOf[T] } - case _ => throw new ScalarFailureException(s"expected to get a single row back") + case x => throw new ScalarFailureException(s"expected to get a single row back, got: $x") } protected class ScalarFailureException(msg: String) extends RuntimeException(msg) diff --git a/community/community-it/cypher-it/src/test/scala/org/neo4j/cypher/ExecutionResultTest.scala b/community/community-it/cypher-it/src/test/scala/org/neo4j/cypher/ExecutionResultTest.scala index 6ec6b8b61ad07..94881c42e019c 100644 --- a/community/community-it/cypher-it/src/test/scala/org/neo4j/cypher/ExecutionResultTest.scala +++ b/community/community-it/cypher-it/src/test/scala/org/neo4j/cypher/ExecutionResultTest.scala @@ -33,13 +33,13 @@ class ExecutionResultTest extends ExecutionEngineFunSuite { "where id(zero) = 0 AND id(one) = 1 AND id(two) = 2 AND id(three) = 3 AND id(four) = 4 AND id(five) = 5 AND id(six) = 6 AND id(seven) = 7 AND id(eight) = 8 AND id(nine) = 9 " + "return zero, one, two, three, four, five, six, seven, eight, nine" - val result = execute(q) + assert(execute(q).columns === columns) - assert( result.columns === columns ) val regex = "zero.*one.*two.*three.*four.*five.*six.*seven.*eight.*nine" val pattern = Pattern.compile(regex) - assertTrue( "Columns did not appear in the expected order: \n" + result.dumpToString(), pattern.matcher(result.dumpToString()).find() ) + val stringDump = graph.execute(q).resultAsString() + assertTrue( "Columns did not appear in the expected order: \n" + stringDump, pattern.matcher(stringDump).find() ) } test("correctLabelStatisticsForCreate") { diff --git a/community/community-it/cypher-it/src/test/scala/org/neo4j/cypher/MergeConcurrencyIT.scala b/community/community-it/cypher-it/src/test/scala/org/neo4j/cypher/MergeConcurrencyIT.scala index 410d2cffcd91e..b63d1b40dc4c6 100644 --- a/community/community-it/cypher-it/src/test/scala/org/neo4j/cypher/MergeConcurrencyIT.scala +++ b/community/community-it/cypher-it/src/test/scala/org/neo4j/cypher/MergeConcurrencyIT.scala @@ -60,7 +60,7 @@ class MergeConcurrencyIT extends ExecutionEngineFunSuite { execute("match (a:Label) with a.id as id, count(*) as c where c > 1 return *") shouldBe empty execute("match (a)-[r1]->(b)<-[r2]-(a) where r1 <> r2 return *") shouldBe empty - val details = "\n" + execute("match (a)-[r]->(b) return a.id, b.id, id(a), id(r), id(b)").dumpToString() + val details = "\n" + graph.execute("match (a)-[r]->(b) return a.id, b.id, id(a), id(r), id(b)").resultAsString() assert(execute(s"match p=(:Label {id:1})-[*..1000]->({id:$nodeCount}) return 1").size === 1, details) } diff --git a/community/community-it/cypher-it/src/test/scala/org/neo4j/cypher/RewindableExecutionResultTest.scala b/community/community-it/cypher-it/src/test/scala/org/neo4j/cypher/RewindableExecutionResultTest.scala deleted file mode 100644 index 1cdea8ce7542d..0000000000000 --- a/community/community-it/cypher-it/src/test/scala/org/neo4j/cypher/RewindableExecutionResultTest.scala +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2002-2018 "Neo4j," - * Neo4j Sweden AB [http://neo4j.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.cypher - -import org.neo4j.graphdb.Node - -class RewindableExecutionResultTest extends ExecutionEngineFunSuite { - test("can do toList twice and get the same result") { - val a = createNode() - val b = createNode() - - val result = execute("match (n) return n") - - result.toList should equal(List(Map("n" -> a), Map("n" -> b))) - result.toList should equal(List(Map("n" -> a), Map("n" -> b))) - } - - test("can dumpToString and then use toList") { - val a = createNode("name" -> "Aslan") - val b = createNode("name" -> "White Queen") - - val result = execute(" match (n) return n") - - assert(List(Map("n" -> a), Map("n" -> b)) === result.toList) - - val textDump = result.dumpToString() - - textDump should include("Aslan") - textDump should include("White Queen") - } - - test("can dumpToString and then use columnAs") { - val a = createNode("name" -> "Aslan") - val b = createNode("name" -> "White Queen") - - val result = execute("match (n) return n") - - assert(List(Map("n" -> a), Map("n" -> b)) === result.toList) - - val nodes = result.columnAs[Node]("n").toList - - nodes should equal(List(a,b)) - } -} diff --git a/community/community-it/cypher-it/src/test/scala/org/neo4j/cypher/RootPlanAcceptanceTest.scala b/community/community-it/cypher-it/src/test/scala/org/neo4j/cypher/RootPlanAcceptanceTest.scala index b15c63c1fc421..6288143c391aa 100644 --- a/community/community-it/cypher-it/src/test/scala/org/neo4j/cypher/RootPlanAcceptanceTest.scala +++ b/community/community-it/cypher-it/src/test/scala/org/neo4j/cypher/RootPlanAcceptanceTest.scala @@ -160,9 +160,9 @@ class RootPlanAcceptanceTest extends ExecutionEngineFunSuite { val runtimeString = runtime.map("runtime=" + _.name).getOrElse("") s"CYPHER $version $plannerString $runtimeString" } - val result = profile(s"$prepend $query") - result.dumpToString() - result.executionPlanDescription() + val result = executeOfficial(s"$prepend PROFILE $query") + result.resultAsString() + result.getExecutionPlanDescription() } } } diff --git a/community/community-it/cypher-it/src/test/scala/org/neo4j/cypher/TxCountsTrackingTestSupport.scala b/community/community-it/cypher-it/src/test/scala/org/neo4j/cypher/TxCountsTrackingTestSupport.scala index 0fb38ebe5add7..dd6c594d4e84c 100644 --- a/community/community-it/cypher-it/src/test/scala/org/neo4j/cypher/TxCountsTrackingTestSupport.scala +++ b/community/community-it/cypher-it/src/test/scala/org/neo4j/cypher/TxCountsTrackingTestSupport.scala @@ -19,13 +19,13 @@ */ package org.neo4j.cypher -import org.neo4j.cypher.internal.runtime.InternalExecutionResult +import org.neo4j.cypher.internal.RewindableExecutionResult import org.opencypher.v9_0.util.test_helpers.{CypherFunSuite, CypherTestSupport} trait TxCountsTrackingTestSupport extends CypherTestSupport { self: CypherFunSuite with GraphDatabaseTestSupport with ExecutionEngineTestSupport => - def executeAndTrackTxCounts(queryText: String, params: (String, Any)*): (InternalExecutionResult, TxCounts) = { + def executeAndTrackTxCounts(queryText: String, params: (String, Any)*): (RewindableExecutionResult, TxCounts) = { val (result, txCounts) = prepareAndTrackTxCounts(execute(queryText, params: _*)) (result, txCounts) } diff --git a/community/cypher/cypher-planner-3.5/src/main/scala/org/neo4j/cypher/internal/compiler/v3_5/Notifications.scala b/community/cypher/cypher-planner-3.5/src/main/scala/org/neo4j/cypher/internal/compiler/v3_5/Notifications.scala index f8c3ec0cd090c..06065426f34a8 100644 --- a/community/cypher/cypher-planner-3.5/src/main/scala/org/neo4j/cypher/internal/compiler/v3_5/Notifications.scala +++ b/community/cypher/cypher-planner-3.5/src/main/scala/org/neo4j/cypher/internal/compiler/v3_5/Notifications.scala @@ -25,6 +25,12 @@ case class SuboptimalIndexForConstainsQueryNotification(label: String, propertyK case class SuboptimalIndexForEndsWithQueryNotification(label: String, propertyKeys: Seq[String]) extends InternalNotification +case object StartUnavailableFallback extends InternalNotification + +case class CreateUniqueUnavailableFallback(position: InputPosition) extends InternalNotification + +case object RulePlannerUnavailableFallbackNotification extends InternalNotification + case object PlannerUnsupportedNotification extends InternalNotification case object RuntimeUnsupportedNotification extends InternalNotification diff --git a/community/cypher/cypher-planner-3.5/src/main/scala/org/neo4j/cypher/internal/compiler/v3_5/ProcedureCallOrSchemaCommandPlanBuilder.scala b/community/cypher/cypher-planner-3.5/src/main/scala/org/neo4j/cypher/internal/compiler/v3_5/ProcedureCallOrSchemaCommandPlanBuilder.scala index e5021d8613e03..b7f81b89b8f15 100644 --- a/community/cypher/cypher-planner-3.5/src/main/scala/org/neo4j/cypher/internal/compiler/v3_5/ProcedureCallOrSchemaCommandPlanBuilder.scala +++ b/community/cypher/cypher-planner-3.5/src/main/scala/org/neo4j/cypher/internal/compiler/v3_5/ProcedureCallOrSchemaCommandPlanBuilder.scala @@ -20,14 +20,14 @@ package org.neo4j.cypher.internal.compiler.v3_5 import org.neo4j.cypher.internal.compiler.v3_5.phases._ -import org.opencypher.v9_0.ast._ -import org.opencypher.v9_0.frontend.phases.{BaseState, Condition, Phase} -import org.opencypher.v9_0.ast.semantics.{SemanticCheckResult, SemanticState} -import org.opencypher.v9_0.util.attribution.SequentialIdGen import org.neo4j.cypher.internal.v3_5.logical.plans import org.neo4j.cypher.internal.v3_5.logical.plans.{LogicalPlan, ResolvedCall} +import org.opencypher.v9_0.ast._ +import org.opencypher.v9_0.ast.semantics.{SemanticCheckResult, SemanticState} import org.opencypher.v9_0.frontend.phases.CompilationPhaseTracer.CompilationPhase import org.opencypher.v9_0.frontend.phases.CompilationPhaseTracer.CompilationPhase.PIPE_BUILDING +import org.opencypher.v9_0.frontend.phases.{BaseState, Condition, Phase} +import org.opencypher.v9_0.util.attribution.SequentialIdGen /** * This planner takes on queries that requires no planning such as procedures and schema commands diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/Compiler.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/Compiler.scala index d56ae0444a1a4..14196f0e8da52 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/Compiler.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/Compiler.scala @@ -23,6 +23,7 @@ import org.neo4j.cypher.CypherException import org.neo4j.kernel.impl.query.TransactionalContext import org.neo4j.values.virtual.MapValue import org.opencypher.v9_0.frontend.phases.CompilationPhaseTracer +import org.opencypher.v9_0.util.InternalNotification /** * Cypher compiler, which compiles pre-parsed queries into executable queries. @@ -30,19 +31,19 @@ import org.opencypher.v9_0.frontend.phases.CompilationPhaseTracer trait Compiler { /** - * Compile pre-parsed query into executable query. + * Compile [[PreParsedQuery]] into [[ExecutableQuery]]. * - * @param preParsedQuery pre-parsed query to convert - * @param tracer compilation tracer to which events of the compilation process are reported + * @param preParsedQuery pre-parsed query to convert + * @param tracer compilation tracer to which events of the compilation process are reported * @param preParsingNotifications notifications from pre-parsing - * @param transactionalContext transactional context to use during compilation (in logical and physical planning) + * @param transactionalContext transactional context to use during compilation (in logical and physical planning) * @throws CypherException public cypher exceptions on compilation problems * @return a compiled and executable query */ @throws[org.neo4j.cypher.CypherException] def compile(preParsedQuery: PreParsedQuery, tracer: CompilationPhaseTracer, - preParsingNotifications: Set[org.neo4j.graphdb.Notification], + preParsingNotifications: Set[InternalNotification], transactionalContext: TransactionalContext, params: MapValue ): ExecutableQuery diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/MasterCompiler.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/MasterCompiler.scala index 9978791b83845..edc07cbead62a 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/MasterCompiler.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/MasterCompiler.scala @@ -21,17 +21,17 @@ package org.neo4j.cypher.internal import java.time.Clock -import org.neo4j.cypher.internal.compiler.v3_5.StatsDivergenceCalculator +import org.neo4j.cypher.internal.compiler.v3_5.{StatsDivergenceCalculator, _} import org.neo4j.cypher.{InvalidArgumentException, _} -import org.neo4j.graphdb.impl.notification.NotificationCode._ -import org.neo4j.graphdb.impl.notification.NotificationDetail.Factory.message +import org.neo4j.graphdb.factory.GraphDatabaseSettings import org.neo4j.kernel.GraphDatabaseQueryService +import org.neo4j.kernel.configuration.Config import org.neo4j.kernel.impl.query.TransactionalContext import org.neo4j.kernel.monitoring.{Monitors => KernelMonitors} -import org.neo4j.values.virtual.MapValue import org.neo4j.logging.LogProvider -import org.opencypher.v9_0.frontend.phases.CompilationPhaseTracer -import org.opencypher.v9_0.util.{InputPosition, SyntaxException => InternalSyntaxException} +import org.neo4j.values.virtual.MapValue +import org.opencypher.v9_0.frontend.phases.{CompilationPhaseTracer, RecordingNotificationLogger} +import org.opencypher.v9_0.util.{DeprecatedStartNotification, SyntaxException => InternalSyntaxException} object MasterCompiler { val DEFAULT_QUERY_CACHE_SIZE: Int = 128 @@ -53,6 +53,8 @@ class MasterCompiler(graph: GraphDatabaseQueryService, logProvider: LogProvider, compilerLibrary: CompilerLibrary) { + import org.neo4j.cypher.internal.MasterCompiler._ + /** * Clear all compiler caches. * @@ -76,7 +78,8 @@ class MasterCompiler(graph: GraphDatabaseQueryService, params: MapValue ): ExecutableQuery = { - var notifications = Set.newBuilder[org.neo4j.graphdb.Notification] + val logger = new RecordingNotificationLogger(Some(preParsedQuery.offset)) + val supportedRuntimes3_1 = Seq(CypherRuntimeOption.interpreted, CypherRuntimeOption.default) val inputPosition = preParsedQuery.offset @@ -85,7 +88,7 @@ class MasterCompiler(graph: GraphDatabaseQueryService, if (config.useErrorsOverWarnings) { throw new InvalidArgumentException("The given query is not currently supported in the selected runtime") } else { - notifications += runtimeUnsupportedNotification(ex, inputPosition) + logger.log(RuntimeUnsupportedNotification) } } } @@ -99,7 +102,7 @@ class MasterCompiler(graph: GraphDatabaseQueryService, def innerCompile(preParsedQuery: PreParsedQuery, params: MapValue): ExecutableQuery = { if ((preParsedQuery.version == CypherVersion.v3_4 || preParsedQuery.version == CypherVersion.v3_5) && preParsedQuery.planner == CypherPlannerOption.rule) { - notifications += rulePlannerUnavailableFallbackNotification(preParsedQuery.offset) + logger.log(RulePlannerUnavailableFallbackNotification) innerCompile(preParsedQuery.copy(version = CypherVersion.v3_1), params) } else if (preParsedQuery.version == CypherVersion.v3_5) { @@ -109,18 +112,18 @@ class MasterCompiler(graph: GraphDatabaseQueryService, preParsedQuery.updateStrategy) try { - compiler3_5.compile(preParsedQuery, tracer, notifications.result(), transactionalContext, params) + compiler3_5.compile(preParsedQuery, tracer, logger.notifications, transactionalContext, params) } catch { case ex: SyntaxException if ex.getMessage.startsWith("CREATE UNIQUE") => val ex3_5 = ex.getCause.asInstanceOf[InternalSyntaxException] - notifications += createUniqueNotification(ex3_5, inputPosition) + logger.log(CreateUniqueUnavailableFallback(ex3_5.pos.get)) assertSupportedRuntime(ex3_5, preParsedQuery.runtime) innerCompile(preParsedQuery.copy(version = CypherVersion.v3_1, runtime = CypherRuntimeOption.interpreted), params) case ex: SyntaxException if ex.getMessage.startsWith("START is deprecated") => val ex3_5 = ex.getCause.asInstanceOf[InternalSyntaxException] - notifications += createStartUnavailableNotification(ex3_5, inputPosition) - notifications += createStartDeprecatedNotification(ex3_5, inputPosition) + logger.log(StartUnavailableFallback) + logger.log(DeprecatedStartNotification(inputPosition, ex.getMessage)) assertSupportedRuntime(ex3_5, preParsedQuery.runtime) innerCompile(preParsedQuery.copy(version = CypherVersion.v3_1, runtime = CypherRuntimeOption.interpreted), params) } @@ -132,7 +135,7 @@ class MasterCompiler(graph: GraphDatabaseQueryService, preParsedQuery.runtime, preParsedQuery.updateStrategy) - compiler.compile(preParsedQuery, tracer, notifications.result(), transactionalContext, params) + compiler.compile(preParsedQuery, tracer, logger.notifications, transactionalContext, params) } } @@ -140,29 +143,33 @@ class MasterCompiler(graph: GraphDatabaseQueryService, innerCompile(preParsedQuery, params) } - private def createStartUnavailableNotification(ex: InternalSyntaxException, inputPosition: InputPosition) = { - val pos = convertInputPosition(ex.pos.getOrElse(inputPosition)) - START_UNAVAILABLE_FALLBACK.notification(pos) + private def getStatisticsDivergenceCalculator: StatsDivergenceCalculator = { + val divergenceThreshold = getSetting(graph, + config => config.get(GraphDatabaseSettings.query_statistics_divergence_threshold).doubleValue(), + DEFAULT_STATISTICS_DIVERGENCE_THRESHOLD) + val targetThreshold = getSetting(graph, + config => config.get(GraphDatabaseSettings.query_statistics_divergence_target).doubleValue(), + DEFAULT_STATISTICS_DIVERGENCE_TARGET) + val minReplanTime = getSetting(graph, + config => config.get(GraphDatabaseSettings.cypher_min_replan_interval).toMillis.longValue(), + DEFAULT_QUERY_PLAN_TTL) + val targetReplanTime = getSetting(graph, + config => config.get(GraphDatabaseSettings.cypher_replan_interval_target).toMillis.longValue(), + DEFAULT_QUERY_PLAN_TARGET) + val divergenceAlgorithm = getSetting(graph, + config => config.get(GraphDatabaseSettings.cypher_replan_algorithm), + DEFAULT_DIVERGENCE_ALGORITHM) + StatsDivergenceCalculator.divergenceCalculatorFor(divergenceAlgorithm, divergenceThreshold, targetThreshold, minReplanTime, targetReplanTime) } - private def createStartDeprecatedNotification(ex: InternalSyntaxException, inputPosition: InputPosition) = { - val pos = convertInputPosition(ex.pos.getOrElse(inputPosition)) - START_DEPRECATED.notification(pos, message("START", ex.getMessage)) + private def getNonIndexedLabelWarningThreshold: Long = { + val setting: (Config) => Long = config => config.get(GraphDatabaseSettings.query_non_indexed_label_warning_threshold).longValue() + getSetting(graph, setting, DEFAULT_NON_INDEXED_LABEL_WARNING_THRESHOLD) } - private def runtimeUnsupportedNotification(ex: InternalSyntaxException, inputPosition: InputPosition) = { - val pos = convertInputPosition(ex.pos.getOrElse(inputPosition)) - RUNTIME_UNSUPPORTED.notification(pos) + private def getSetting[A](gds: GraphDatabaseQueryService, configLookup: Config => A, default: A): A = gds match { + // TODO: Cypher should not be pulling out components from casted interfaces, it should ask for Config as a dep + case gdbApi:GraphDatabaseQueryService => configLookup(gdbApi.getDependencyResolver.resolveDependency(classOf[Config])) + case _ => default } - - private def createUniqueNotification(ex: InternalSyntaxException, inputPosition: InputPosition) = { - val pos = convertInputPosition(ex.pos.getOrElse(inputPosition)) - CREATE_UNIQUE_UNAVAILABLE_FALLBACK.notification(pos) - } - - private def rulePlannerUnavailableFallbackNotification(offset: InputPosition) = - RULE_PLANNER_UNAVAILABLE_FALLBACK.notification(convertInputPosition(offset)) - - private def convertInputPosition(offset: InputPosition) = - new org.neo4j.graphdb.InputPosition(offset.offset, offset.line, offset.column) } diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/BasePlanner.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/BasePlanner.scala index a778beb1237aa..0045410dabba7 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/BasePlanner.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/BasePlanner.scala @@ -76,7 +76,7 @@ abstract class BasePlanner[STATEMENT <: AnyRef, PARSED_STATE <: AnyRef]( } protected def createReusabilityState(logicalPlanState: LogicalPlanState, - planContext: PlanContext): ReusabilityState = { + planContext: PlanContext): ReusabilityState = { if (ProcedureCallOrSchemaCommandRuntime .logicalToExecutable diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/ClosingExecutionResult.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/ClosingExecutionResult.scala index 19674ed47e1ff..ee468de02bf2b 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/ClosingExecutionResult.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/ClosingExecutionResult.scala @@ -20,10 +20,11 @@ package org.neo4j.cypher.internal.compatibility import java.io.PrintWriter +import java.util import org.neo4j.cypher.exceptionHandler.RunSafely -import org.neo4j.cypher.internal.runtime.{ExecutionMode, InternalExecutionResult, InternalQueryType} import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription +import org.neo4j.cypher.internal.runtime._ import org.neo4j.cypher.result.QueryResult.QueryResultVisitor import org.neo4j.graphdb import org.neo4j.graphdb.Result.ResultVisitor @@ -31,168 +32,129 @@ import org.neo4j.graphdb.{Notification, ResourceIterator} import org.neo4j.kernel.api.query.ExecutingQuery import org.neo4j.kernel.impl.query.QueryExecutionMonitor +/** + * Ensures execution results are closed. This is tricky because we try to be smart about + * closing results automatically when + * + * 1) all result rows have been seen through iterator + * 2) all result rows have been seen through visitor + * 3) all result rows have been seen through dumpToString + * 4) any operator throws an exception + * + * In addition we also have special handling for suppressing exceptions thrown on close() + * after responding to a 4). + * + * Finally this class report to the [[innerMonitor]] when the query is closed. + * + * @param query metadata about the executing query + * @param inner the actual result + * @param runSafely RunSafely which converts any exception into the public exception space (subtypes of org.neo4j.cypher.CypherException) + * @param innerMonitor monitor to report closing of the query to + */ class ClosingExecutionResult(val query: ExecutingQuery, val inner: InternalExecutionResult, runSafely: RunSafely) (implicit innerMonitor: QueryExecutionMonitor) extends InternalExecutionResult { - private val monitor = OnlyOnceQueryExecutionMonitor(innerMonitor) - - // Queries with no columns are queries that do not RETURN anything. - // In these cases, it's safe to close the results eagerly - if (inner.columns.isEmpty) - runSafely { - closeIfEmpty() - } - - override def planDescriptionRequested: Boolean = runSafely { - inner.planDescriptionRequested - } - - override def javaIterator: graphdb.ResourceIterator[java.util.Map[String, Any]] = { - val innerJavaIterator = inner.javaIterator - - runSafely { - closeIfEmpty() - } - - new graphdb.ResourceIterator[java.util.Map[String, Any]] { - def close() = runSafely { - endQueryExecution() - innerJavaIterator.close() - } + self => - def next() = runSafely { - val result = innerJavaIterator.next - closeIfEmpty() - result - } + private val monitor = OnlyOnceQueryExecutionMonitor(innerMonitor) - def hasNext = runSafely { - closeIfEmpty() - innerJavaIterator.hasNext - } + override def javaIterator: graphdb.ResourceIterator[java.util.Map[String, AnyRef]] = { + safely { + val innerIterator = inner.javaIterator + closeIfEmpty(innerIterator) - def remove() = runSafely { - innerJavaIterator.remove() - } - } - } + new graphdb.ResourceIterator[java.util.Map[String, AnyRef]] { + def next(): util.Map[String, AnyRef] = safely { + val result = innerIterator.next + closeIfEmpty(innerIterator) + result + } - override def columnAs[T](column: String): Iterator[T] = runSafely { - new Iterator[T] { - private val _inner = inner.columnAs[T](column) + def hasNext: Boolean = safely { + closeIfEmpty(innerIterator) + innerIterator.hasNext + } - override def hasNext: Boolean = runSafely { - closeIfEmpty() - _inner.hasNext - } + def close(): Unit = self.close() - override def next(): T = runSafely { - val result = _inner.next() - closeIfEmpty() - result + def remove(): Unit = safely { + innerIterator.remove() + } } } } - override def fieldNames() = runSafely { - inner.fieldNames() - } - + override def fieldNames(): Array[String] = safely { inner.fieldNames() } - override def queryStatistics() = runSafely { inner.queryStatistics() } + override def queryStatistics(): QueryStatistics = safely { inner.queryStatistics() } - override def dumpToString(writer: PrintWriter) = runSafely { - inner.dumpToString(writer) - closeIfEmpty() - } + override def dumpToString(writer: PrintWriter): Unit = safelyAndClose { inner.dumpToString(writer) } - override def dumpToString() = runSafely { - val result = inner.dumpToString() - closeIfEmpty() - result - } + override def dumpToString(): String = safelyAndClose { inner.dumpToString() } - override def javaColumnAs[T](column: String) = runSafely { - val _inner = inner.javaColumnAs[T](column) - new ResourceIterator[T] { + override def javaColumnAs[T](column: String): ResourceIterator[T] = + safely { + val _inner = inner.javaColumnAs[T](column) + new ResourceIterator[T] { - override def hasNext: Boolean = runSafely { - closeIfEmpty() - _inner.hasNext - } + override def hasNext: Boolean = + safely { + closeIfEmpty(_inner) + _inner.hasNext + } - override def next(): T = runSafely { - val result = _inner.next() - closeIfEmpty() - result - } + override def next(): T = + safely { + val result = _inner.next() + closeIfEmpty(_inner) + result + } - override def close(): Unit = runSafely { - _inner.close() - endQueryExecution() + override def close(): Unit = self.close() } } - } override def executionPlanDescription(): InternalPlanDescription = - runSafely { + safely { inner.executionPlanDescription() } - override def close(): Unit = runSafely { + override def close(): Unit = runSafely({ inner.close() - endQueryExecution() - } + })(t => monitor.endSuccess(query)) - override def next(): Map[String, Any] = runSafely { - val result = inner.next() - closeIfEmpty() - result - } + override def queryType: InternalQueryType = safely { inner.queryType } - override def hasNext: Boolean = runSafely { - val next = inner.hasNext - if (!next) { - endQueryExecution() - } - next - } + override def notifications: Iterable[Notification] = safely { inner.notifications } - override def queryType: InternalQueryType = runSafely { - inner.queryType - } + override def accept[EX <: Exception](visitor: ResultVisitor[EX]): Unit = + safelyAndClose { + inner.accept(visitor) + } - override def notifications: Iterable[Notification] = runSafely { inner.notifications } + override def accept[EX <: Exception](visitor: QueryResultVisitor[EX]): Unit = + safelyAndClose { + inner.accept(visitor) + } - override def accept[EX <: Exception](visitor: ResultVisitor[EX]): Unit = runSafely { - inner.accept(visitor) - endQueryExecution() - } + override def executionMode: ExecutionMode = safely { inner.executionMode } - override def accept[EX <: Exception](visitor: QueryResultVisitor[EX]): Unit = runSafely { - inner.accept(visitor) - endQueryExecution() - } + override def toString: String = runSafely { inner.toString } - override def toString(): String = runSafely { - inner.toString() - } + // HELPERS - private def closeIfEmpty(): Unit = { - if (!inner.hasNext) { - endQueryExecution() - } - } - - private def endQueryExecution() = { - monitor.endSuccess(query) // this method is expected to be idempotent - } + private def safely[T](body: => T): T = runSafely(body)(closeOnError) - override def executionMode: ExecutionMode = runSafely(inner.executionMode) + private def safelyAndClose[T](body: => T): T = + runSafely({ + val x = body + close(true) + x + })(closeOnError) - override def withNotifications(added: Notification*): InternalExecutionResult = - new ClosingExecutionResult(query, inner, runSafely) { - override def notifications: Iterable[Notification] = super.notifications ++ added + private def closeIfEmpty(iterator: java.util.Iterator[_]): Unit = + if (!iterator.hasNext) { + close() } } diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/CypherCurrentCompiler.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/CypherCurrentCompiler.scala index 07dca004dec8a..aa731142ef4b8 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/CypherCurrentCompiler.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/CypherCurrentCompiler.scala @@ -22,10 +22,13 @@ package org.neo4j.cypher.internal.compatibility import org.neo4j.cypher.exceptionHandler.runSafely import org.neo4j.cypher.internal._ import org.neo4j.cypher.internal.compatibility.v3_5.ExceptionTranslatingQueryContext -import org.neo4j.cypher.internal.compatibility.v3_5.runtime.RuntimeName -import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.{ExecutionPlan => ExecutionPlan_v3_5} +import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.{StandardInternalExecutionResult, ExecutionPlan => ExecutionPlan_v3_5} +import org.neo4j.cypher.internal.compatibility.v3_5.runtime.helpers.InternalWrapping.asKernelNotification +import org.neo4j.cypher.internal.compatibility.v3_5.runtime.profiler.PlanDescriptionBuilder +import org.neo4j.cypher.internal.compatibility.v3_5.runtime.{ExplainExecutionResult, RuntimeName} import org.neo4j.cypher.internal.compiler.v3_5.phases.LogicalPlanState import org.neo4j.cypher.internal.javacompat.ExecutionResult +import org.neo4j.cypher.internal.planner.v3_5.spi.PlanningAttributes.Cardinalities import org.neo4j.cypher.internal.runtime.interpreted.TransactionBoundQueryContext.IndexSearchMonitor import org.neo4j.cypher.internal.runtime.interpreted.{TransactionBoundQueryContext, TransactionalContextWrapper} import org.neo4j.cypher.internal.runtime.{ExecutableQuery => _, _} @@ -36,13 +39,14 @@ import org.neo4j.kernel.api.query.{CompilerInfo, ExplicitIndexUsage, SchemaIndex import org.neo4j.kernel.impl.query.{QueryExecutionMonitor, TransactionalContext} import org.neo4j.kernel.monitoring.{Monitors => KernelMonitors} import org.neo4j.values.virtual.MapValue -import org.opencypher.v9_0.frontend.phases.CompilationPhaseTracer +import org.opencypher.v9_0.frontend.phases.{CompilationPhaseTracer, RecordingNotificationLogger} +import org.opencypher.v9_0.util.{InternalNotification, TaskCloser} import scala.collection.JavaConverters._ /** - * Composite compiler, which uses a CypherPlanner and CypherRuntime to compile - * a preparsed query into a CacheableExecutableQuery. + * Composite [[Compiler]], which uses a [[CypherPlanner]] and [[CypherRuntime]] to compile + * a preparsed query into a [[ExecutableQuery]]. * * @param planner the planner * @param runtime the runtime @@ -57,7 +61,7 @@ case class CypherCurrentCompiler[CONTEXT <: RuntimeContext](planner: CypherPlann ) extends org.neo4j.cypher.internal.Compiler { /** - * Compile pre-parsed query into executable query. + * Compile [[PreParsedQuery]] into [[ExecutableQuery]]. * * @param preParsedQuery pre-parsed query to convert * @param tracer compilation tracer to which events of the compilation process are reported @@ -68,12 +72,15 @@ case class CypherCurrentCompiler[CONTEXT <: RuntimeContext](planner: CypherPlann */ override def compile(preParsedQuery: PreParsedQuery, tracer: CompilationPhaseTracer, - preParsingNotifications: Set[Notification], + preParsingNotifications: Set[InternalNotification], transactionalContext: TransactionalContext, - params: MapValue): ExecutableQuery = { + params: MapValue + ): ExecutableQuery = { + + val planningNotificationLogger = new RecordingNotificationLogger(Some(preParsedQuery.offset)) val logicalPlanResult = - planner.parseAndPlan(preParsedQuery, tracer, preParsingNotifications, transactionalContext, params) + planner.parseAndPlan(preParsedQuery, tracer, planningNotificationLogger, transactionalContext, params) val planState = logicalPlanResult.logicalPlanState val logicalPlan = planState.logicalPlan @@ -88,15 +95,20 @@ case class CypherCurrentCompiler[CONTEXT <: RuntimeContext](planner: CypherPlann val executionPlan3_5 = runtime.compileToExecutable(planState, runtimeContext) new CypherExecutableQuery( + logicalPlan, + runtimeContext.readOnly, + logicalPlanResult.logicalPlanState.cardinalities, executionPlan3_5, - preParsingNotifications, + preParsingNotifications.map(asKernelNotification(None)), + planningNotificationLogger.notifications.map(asKernelNotification(planningNotificationLogger.offset)), logicalPlanResult.reusability, logicalPlanResult.paramNames, logicalPlanResult.extractedParams, - buildCompilerInfo(logicalPlan, executionPlan3_5.runtimeName)) + buildCompilerInfo(logicalPlan, executionPlan3_5.runtimeName), + queryType) } - def buildCompilerInfo(logicalPlan: LogicalPlan, runtimeName: RuntimeName): CompilerInfo = + private def buildCompilerInfo(logicalPlan: LogicalPlan, runtimeName: RuntimeName): CompilerInfo = new CompilerInfo(planner.name.name, runtimeName.name, logicalPlan.indexUsage.map { case SchemaIndexSeekUsage(identifier, labelId, label, propertyKeys) => new SchemaIndexUsage(identifier, labelId, label, propertyKeys: _*) case SchemaIndexScanUsage(identifier, labelId, label, propertyKey) => new SchemaIndexUsage(identifier, labelId, label, propertyKey) @@ -126,16 +138,21 @@ case class CypherCurrentCompiler[CONTEXT <: RuntimeContext](planner: CypherPlann case _ => Array() } - protected class CypherExecutableQuery(inner: ExecutionPlan_v3_5, - preParsingNotifications: Set[org.neo4j.graphdb.Notification], + protected class CypherExecutableQuery(logicalPlan: LogicalPlan, + readOnly: Boolean, + cardinalities: Cardinalities, + executionPlan: ExecutionPlan_v3_5, + preParsingNotifications: Set[Notification], + planningNotifications: Set[Notification], reusabilityState: ReusabilityState, override val paramNames: Seq[String], override val extractedParams: MapValue, - override val compilerInfo: CompilerInfo) extends ExecutableQuery { + override val compilerInfo: CompilerInfo, + queryType: InternalQueryType) extends ExecutableQuery { private val searchMonitor = kernelMonitors.newMonitor(classOf[IndexSearchMonitor]) - private def queryContext(transactionalContext: TransactionalContext) = { + private def getQueryContext(transactionalContext: TransactionalContext) = { val ctx = new TransactionBoundQueryContext(TransactionalContextWrapper(transactionalContext))(searchMonitor) new ExceptionTranslatingQueryContext(ctx) } @@ -149,14 +166,44 @@ case class CypherCurrentCompiler[CONTEXT <: RuntimeContext](planner: CypherPlann } runSafely { - val context = queryContext(transactionalContext) - - val innerResult: InternalExecutionResult = inner.run(context, innerExecutionMode, params) - new ExecutionResult(new ClosingExecutionResult( - transactionalContext.executingQuery(), - innerResult.withNotifications(preParsingNotifications.toSeq: _*), - runSafely - )(kernelMonitors.newMonitor(classOf[QueryExecutionMonitor]))) + val queryContext = getQueryContext(transactionalContext) + + val planDescriptionBuilder = + new PlanDescriptionBuilder(logicalPlan, planner.name, readOnly, cardinalities, executionPlan.runtimeName) + + val taskCloser = new TaskCloser + taskCloser.addTask(queryContext.transactionalContext.close) + taskCloser.addTask(queryContext.resources.close) + + val internalExecutionResult = + if (innerExecutionMode == ExplainMode) { + taskCloser.close(success = true) + val columns = columnNames(logicalPlan) + + ExplainExecutionResult(columns, + planDescriptionBuilder.explain(), + queryType, + preParsingNotifications ++ planningNotifications) + } else { + + val doProfile = innerExecutionMode == ProfileMode + val runtimeResult = executionPlan.run(queryContext, doProfile, params) + + new StandardInternalExecutionResult(queryContext, + executionPlan.runtimeName, + runtimeResult, + taskCloser, + queryType, + preParsingNotifications, + innerExecutionMode, + planDescriptionBuilder) + } + + new ExecutionResult( + new ClosingExecutionResult( + transactionalContext.executingQuery(), + internalExecutionResult, + runSafely)(kernelMonitors.newMonitor(classOf[QueryExecutionMonitor]))) } } diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/CypherPlanner.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/CypherPlanner.scala index a65cb5b567584..5393a5dcadcf6 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/CypherPlanner.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/CypherPlanner.scala @@ -25,19 +25,19 @@ import org.neo4j.cypher.internal.{PreParsedQuery, ReusabilityState} import org.neo4j.kernel.impl.query.TransactionalContext import org.neo4j.values.virtual.MapValue import org.opencypher.v9_0.frontend.PlannerName -import org.opencypher.v9_0.frontend.phases.CompilationPhaseTracer +import org.opencypher.v9_0.frontend.phases.{CompilationPhaseTracer, InternalNotificationLogger} /** - * Cypher planner, which parses and plans pre-parsed queries into a logical plan. + * Cypher planner, which parses and plans a [[PreParsedQuery]] into a [[LogicalPlanResult]]. */ trait CypherPlanner { /** * Compile pre-parsed query into a logical plan. * - * @param preParsedQuery pre-parsed query to convert - * @param tracer tracer to which events of the parsing and planning are reported - * @param preParsingNotifications notifications from pre-parsing + * @param preParsedQuery pre-parsed query to convert + * @param tracer tracer to which events of the parsing and planning are reported + * @param notifications notifications from pre-parsing * @param transactionalContext transactional context to use during parsing and planning * @throws CypherException public cypher exceptions on compilation problems * @return a logical plan result @@ -45,7 +45,7 @@ trait CypherPlanner { @throws[org.neo4j.cypher.CypherException] def parseAndPlan(preParsedQuery: PreParsedQuery, tracer: CompilationPhaseTracer, - preParsingNotifications: Set[org.neo4j.graphdb.Notification], + notifications: InternalNotificationLogger, transactionalContext: TransactionalContext, params: MapValue ): LogicalPlanResult diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/InterpretedRuntime.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/InterpretedRuntime.scala index ef40401fbad6a..e8704630ac5d1 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/InterpretedRuntime.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/InterpretedRuntime.scala @@ -21,15 +21,14 @@ package org.neo4j.cypher.internal.compatibility import org.neo4j.cypher.internal.compatibility.v3_5.runtime._ import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan._ -import org.neo4j.cypher.internal.compatibility.v3_5.runtime.profiler.Profiler +import org.neo4j.cypher.internal.compatibility.v3_5.runtime.profiler.{InterpretedProfileInformation, Profiler} import org.neo4j.cypher.internal.compiler.v3_5.phases.LogicalPlanState -import org.neo4j.cypher.internal.planner.v3_5.spi.PlanningAttributes.Cardinalities +import org.neo4j.cypher.internal.runtime.QueryContext import org.neo4j.cypher.internal.runtime.interpreted.UpdateCountingQueryContext import org.neo4j.cypher.internal.runtime.interpreted.commands.convert.{CommunityExpressionConverter, ExpressionConverters} import org.neo4j.cypher.internal.runtime.interpreted.pipes.PipeExecutionBuilderContext -import org.neo4j.cypher.internal.runtime.{ExecutionMode, InternalExecutionResult, ProfileMode, QueryContext} +import org.neo4j.cypher.result.RuntimeResult import org.neo4j.values.virtual.MapValue -import org.opencypher.v9_0.frontend.PlannerName import org.opencypher.v9_0.frontend.phases.InternalNotificationLogger import org.opencypher.v9_0.util.PeriodicCommitInOpenTransactionException @@ -41,7 +40,7 @@ object InterpretedRuntime extends CypherRuntime[RuntimeContext] { val executionPlanBuilder = new PipeExecutionPlanBuilder( expressionConverters = converters, pipeBuilderFactory = InterpretedPipeBuilderFactory) - val pipeBuildContext = PipeExecutionBuilderContext(state.semanticTable(), context.readOnly, cardinalities) + val pipeBuildContext = PipeExecutionBuilderContext(state.semanticTable(), context.readOnly) val pipe = executionPlanBuilder.build(logicalPlan)(pipeBuildContext, context.tokenContext) val periodicCommitInfo = state.periodicCommit.map(x => PeriodicCommitInfo(x.batchSize)) val columns = state.statement().returnColumns @@ -50,10 +49,8 @@ object InterpretedRuntime extends CypherRuntime[RuntimeContext] { new InterpretedExecutionPlan(periodicCommitInfo, resultBuilderFactory, context.notificationLogger, - state.plannerName, InterpretedRuntimeName, - context.readOnly, - cardinalities) + context.readOnly) } /** @@ -63,16 +60,14 @@ object InterpretedRuntime extends CypherRuntime[RuntimeContext] { class InterpretedExecutionPlan(periodicCommit: Option[PeriodicCommitInfo], resultBuilderFactory: ExecutionResultBuilderFactory, notificationLogger: InternalNotificationLogger, - plannerName: PlannerName, override val runtimeName: RuntimeName, - readOnly: Boolean, - cardinalities: Cardinalities) extends ExecutionPlan { + readOnly: Boolean) extends ExecutionPlan { - override def run(queryContext: QueryContext, planType: ExecutionMode, params: MapValue): InternalExecutionResult = { + override def run(queryContext: QueryContext, doProfile: Boolean, params: MapValue): RuntimeResult = { val builder = resultBuilderFactory.create() - val profiling = planType == ProfileMode - val builderContext = if (!readOnly || profiling) new UpdateCountingQueryContext(queryContext) else queryContext + val profileInformation = new InterpretedProfileInformation + val builderContext = if (!readOnly || doProfile) new UpdateCountingQueryContext(queryContext) else queryContext builder.setQueryContext(builderContext) @@ -82,10 +77,13 @@ object InterpretedRuntime extends CypherRuntime[RuntimeContext] { builder.setLoadCsvPeriodicCommitObserver(periodicCommit.get.batchRowCount) } - if (profiling) - builder.setPipeDecorator(new Profiler(queryContext.transactionalContext.databaseInfo)) + if (doProfile) + builder.setPipeDecorator(new Profiler(queryContext.transactionalContext.databaseInfo, profileInformation)) - builder.build(planType, params, notificationLogger, plannerName, runtimeName, readOnly, cardinalities) + builder.build(params, + notificationLogger, + readOnly, + profileInformation) } } } diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/ProcedureCallOrSchemaCommandRuntime.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/ProcedureCallOrSchemaCommandRuntime.scala index 597c70f752d5f..b23866ceffdcb 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/ProcedureCallOrSchemaCommandRuntime.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/ProcedureCallOrSchemaCommandRuntime.scala @@ -20,8 +20,7 @@ package org.neo4j.cypher.internal.compatibility import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.ExecutionPlan -import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.procs.{ProcedureCallExecutionPlan, PureSideEffectExecutionPlan} -import org.neo4j.cypher.internal.compatibility.v3_5.runtime.helpers.InternalWrapping.asKernelNotification +import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.procs.{ProcedureCallExecutionPlan, SchemaWriteExecutionPlan} import org.neo4j.cypher.internal.compiler.v3_5.phases.LogicalPlanState import org.neo4j.cypher.internal.compiler.v3_5.planner.CantCompileQueryException import org.neo4j.cypher.internal.planner.v3_5.spi.IndexDescriptor @@ -29,7 +28,6 @@ import org.neo4j.cypher.internal.runtime.interpreted.commands.convert.{Community import org.neo4j.cypher.internal.runtime.{InternalQueryType, ProcedureCallMode, QueryContext, SCHEMA_WRITE} import org.neo4j.cypher.internal.v3_5.logical.plans._ import org.opencypher.v9_0.expressions.{LabelName, PropertyKeyName, RelTypeName} -import org.opencypher.v9_0.frontend.phases.InternalNotificationLogger import org.opencypher.v9_0.util.{LabelId, PropertyKeyId} /** @@ -43,7 +41,7 @@ object ProcedureCallOrSchemaCommandRuntime extends CypherRuntime[RuntimeContext] s"Plan is not a procedure call or schema command: ${unknownPlan.getClass.getSimpleName}") } - logicalToExecutable.applyOrElse(state.maybeLogicalPlan.get, throwCantCompile)(context.notificationLogger) + logicalToExecutable.applyOrElse(state.maybeLogicalPlan.get, throwCantCompile) } def queryType(logicalPlan: LogicalPlan): Option[InternalQueryType] = @@ -55,78 +53,77 @@ object ProcedureCallOrSchemaCommandRuntime extends CypherRuntime[RuntimeContext] } } else None - val logicalToExecutable: PartialFunction[LogicalPlan, InternalNotificationLogger => ExecutionPlan] = { + val logicalToExecutable: PartialFunction[LogicalPlan, ExecutionPlan] = { // Global call: CALL foo.bar.baz("arg1", 2) case StandAloneProcedureCall(signature, args, types, indices) => val converters = new ExpressionConverters(CommunityExpressionConverter) - logger => ProcedureCallExecutionPlan(signature, args, types, indices, - logger.notifications.map(asKernelNotification(logger.offset)), converters) + ProcedureCallExecutionPlan(signature, args, types, indices, converters) // CREATE CONSTRAINT ON (node:Label) ASSERT (node.prop1,node.prop2) IS NODE KEY case CreateNodeKeyConstraint(_, label, props) => - logger => PureSideEffectExecutionPlan("CreateNodeKeyConstraint", SCHEMA_WRITE, ctx => { - val propertyKeyIds = props.map(p => propertyToId(ctx)(p.propertyKey)) - ctx.createNodeKeyConstraint(IndexDescriptor(labelToId(ctx)(label), propertyKeyIds)) - }) + SchemaWriteExecutionPlan("CreateNodeKeyConstraint", ctx => { + val propertyKeyIds = props.map(p => propertyToId(ctx)(p.propertyKey)) + ctx.createNodeKeyConstraint(IndexDescriptor(labelToId(ctx)(label), propertyKeyIds)) + }) // DROP CONSTRAINT ON (node:Label) ASSERT (node.prop1,node.prop2) IS NODE KEY case DropNodeKeyConstraint(label, props) => - logger => PureSideEffectExecutionPlan("DropNodeKeyConstraint", SCHEMA_WRITE, ctx => { - val propertyKeyIds = props.map(p => propertyToId(ctx)(p.propertyKey)) - ctx.dropNodeKeyConstraint(IndexDescriptor(labelToId(ctx)(label), propertyKeyIds)) - }) + SchemaWriteExecutionPlan("DropNodeKeyConstraint", ctx => { + val propertyKeyIds = props.map(p => propertyToId(ctx)(p.propertyKey)) + ctx.dropNodeKeyConstraint(IndexDescriptor(labelToId(ctx)(label), propertyKeyIds)) + }) // CREATE CONSTRAINT ON (node:Label) ASSERT node.prop IS UNIQUE // CREATE CONSTRAINT ON (node:Label) ASSERT (node.prop1,node.prop2) IS UNIQUE case CreateUniquePropertyConstraint(_, label, props) => - logger => PureSideEffectExecutionPlan("CreateUniqueConstraint", SCHEMA_WRITE, ctx => { - val propertyKeyIds = props.map(p => propertyToId(ctx)(p.propertyKey)) - ctx.createUniqueConstraint(IndexDescriptor(labelToId(ctx)(label), propertyKeyIds)) - }) + SchemaWriteExecutionPlan("CreateUniqueConstraint", ctx => { + val propertyKeyIds = props.map(p => propertyToId(ctx)(p.propertyKey)) + ctx.createUniqueConstraint(IndexDescriptor(labelToId(ctx)(label), propertyKeyIds)) + }) // DROP CONSTRAINT ON (node:Label) ASSERT node.prop IS UNIQUE // DROP CONSTRAINT ON (node:Label) ASSERT (node.prop1,node.prop2) IS UNIQUE case DropUniquePropertyConstraint(label, props) => - logger => PureSideEffectExecutionPlan("DropUniqueConstraint", SCHEMA_WRITE, ctx => { - val propertyKeyIds = props.map(p => propertyToId(ctx)(p.propertyKey)) - ctx.dropUniqueConstraint(IndexDescriptor(labelToId(ctx)(label), propertyKeyIds)) - }) + SchemaWriteExecutionPlan("DropUniqueConstraint", ctx => { + val propertyKeyIds = props.map(p => propertyToId(ctx)(p.propertyKey)) + ctx.dropUniqueConstraint(IndexDescriptor(labelToId(ctx)(label), propertyKeyIds)) + }) // CREATE CONSTRAINT ON (node:Label) ASSERT node.prop EXISTS case CreateNodePropertyExistenceConstraint(label, prop) => - logger => PureSideEffectExecutionPlan("CreateNodePropertyExistenceConstraint", SCHEMA_WRITE, ctx => { - (ctx.createNodePropertyExistenceConstraint _).tupled(labelProp(ctx)(label, prop.propertyKey)) - }) + SchemaWriteExecutionPlan("CreateNodePropertyExistenceConstraint", ctx => { + (ctx.createNodePropertyExistenceConstraint _).tupled(labelProp(ctx)(label, prop.propertyKey)) + }) // DROP CONSTRAINT ON (node:Label) ASSERT node.prop EXISTS case DropNodePropertyExistenceConstraint(label, prop) => - logger => PureSideEffectExecutionPlan("DropNodePropertyExistenceConstraint", SCHEMA_WRITE, ctx => { - (ctx.dropNodePropertyExistenceConstraint _).tupled(labelProp(ctx)(label, prop.propertyKey)) - }) + SchemaWriteExecutionPlan("DropNodePropertyExistenceConstraint", ctx => { + (ctx.dropNodePropertyExistenceConstraint _).tupled(labelProp(ctx)(label, prop.propertyKey)) + }) // CREATE CONSTRAINT ON ()-[r:R]-() ASSERT r.prop EXISTS case CreateRelationshipPropertyExistenceConstraint(relType, prop) => - logger => PureSideEffectExecutionPlan("CreateRelationshipPropertyExistenceConstraint", SCHEMA_WRITE, ctx => { - (ctx.createRelationshipPropertyExistenceConstraint _).tupled(typeProp(ctx)(relType, prop.propertyKey)) - }) + SchemaWriteExecutionPlan("CreateRelationshipPropertyExistenceConstraint", ctx => { + (ctx.createRelationshipPropertyExistenceConstraint _).tupled(typeProp(ctx)(relType, prop.propertyKey)) + }) // DROP CONSTRAINT ON ()-[r:R]-() ASSERT r.prop EXISTS case DropRelationshipPropertyExistenceConstraint(relType, prop) => - logger => PureSideEffectExecutionPlan("DropRelationshipPropertyExistenceConstraint", SCHEMA_WRITE, ctx => { - (ctx.dropRelationshipPropertyExistenceConstraint _).tupled(typeProp(ctx)(relType, prop.propertyKey)) - }) + SchemaWriteExecutionPlan("DropRelationshipPropertyExistenceConstraint", ctx => { + (ctx.dropRelationshipPropertyExistenceConstraint _).tupled(typeProp(ctx)(relType, prop.propertyKey)) + }) // CREATE INDEX ON :LABEL(prop) case CreateIndex(label, props) => - logger => PureSideEffectExecutionPlan("CreateIndex", SCHEMA_WRITE, ctx => { - ctx.addIndexRule(IndexDescriptor(labelToId(ctx)(label), propertiesToIds(ctx)(props))) - }) + SchemaWriteExecutionPlan("CreateIndex", ctx => { + ctx.addIndexRule(IndexDescriptor(labelToId(ctx)(label), propertiesToIds(ctx)(props))) + }) // DROP INDEX ON :LABEL(prop) case DropIndex(label, props) => - logger => PureSideEffectExecutionPlan("DropIndex", SCHEMA_WRITE, ctx => { - ctx.dropIndexRule(IndexDescriptor(labelToId(ctx)(label), propertiesToIds(ctx)(props))) - }) + SchemaWriteExecutionPlan("DropIndex", ctx => { + ctx.dropIndexRule(IndexDescriptor(labelToId(ctx)(label), propertiesToIds(ctx)(props))) + }) } implicit private def labelToId(ctx: QueryContext)(label: LabelName): LabelId = diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v2_3/Cypher23Compiler.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v2_3/Cypher23Compiler.scala index 05662a8ba8499..d9052ec2bd3b7 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v2_3/Cypher23Compiler.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v2_3/Cypher23Compiler.scala @@ -25,6 +25,7 @@ import org.neo4j.cypher.CypherExecutionMode import org.neo4j.cypher.internal._ import org.neo4j.cypher.internal.compatibility._ import org.neo4j.cypher.internal.compatibility.v2_3.helpers.as2_3 +import org.neo4j.cypher.internal.compatibility.v3_5.runtime.helpers.InternalWrapping.{asKernelNotification => asKernelNotification3_5} import org.neo4j.cypher.internal.compiler.v2_3 import org.neo4j.cypher.internal.compiler.v2_3.executionplan.{EntityAccessor, ExecutionPlan => ExecutionPlan_v2_3} import org.neo4j.cypher.internal.compiler.v2_3.spi.{PlanContext, QueryContext} @@ -45,6 +46,7 @@ import org.neo4j.logging.Log import org.neo4j.values.AnyValue import org.neo4j.values.virtual.MapValue import org.opencypher.v9_0.frontend.phases.CompilationPhaseTracer +import org.opencypher.v9_0.util.InternalNotification import scala.collection.mutable @@ -92,7 +94,13 @@ trait Cypher23Compiler extends CachingPlanner[PreparedQuery] with Compiler { new ExecutionResult( new ClosingExecutionResult( query, - new ExecutionResultWrapper(innerResult, inner.plannerUsed, inner.runtimeUsed, preParsingNotifications, Some(offSet)), + new ExecutionResultWrapper( + innerResult, + inner.plannerUsed, + inner.runtimeUsed, + preParsingNotifications, + Some(offSet) + ), exceptionHandler.runSafely) ) } @@ -122,7 +130,7 @@ trait Cypher23Compiler extends CachingPlanner[PreparedQuery] with Compiler { override def compile(preParsedQuery: PreParsedQuery, tracer: CompilationPhaseTracer, - preParsingNotifications: Set[org.neo4j.graphdb.Notification], + preParsingNotifications: Set[InternalNotification], transactionalContext: TransactionalContext, params: MapValue ): ExecutableQuery = { @@ -143,7 +151,7 @@ trait Cypher23Compiler extends CachingPlanner[PreparedQuery] with Compiler { executionPlan2_3.notifications(planContext).foreach(notificationLogger += _) new Cypher23ExecutableQuery( executionPlan2_3, - preParsingNotifications, + preParsingNotifications.map(asKernelNotification3_5(None)), position2_3, Seq.empty[String], ValueConversion.asValues(extractedParameters)) diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v2_3/ExecutionResultWrapper.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v2_3/ExecutionResultWrapper.scala index 474fa5fdc6221..c33ca2e8eb7d7 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v2_3/ExecutionResultWrapper.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v2_3/ExecutionResultWrapper.scala @@ -48,13 +48,7 @@ import scala.collection.JavaConverters._ object ExecutionResultWrapper { - def unapply(v: Any): Option[(InternalExecutionResult, PlannerName, RuntimeName, Set[org.neo4j.graphdb.Notification], Option[v2_3.InputPosition])] = v match { - case closing: ClosingExecutionResult => unapply(closing.inner) - case wrapper: ExecutionResultWrapper => Some((wrapper.inner, wrapper.planner, wrapper.runtime, wrapper.preParsingNotifications, wrapper.offset)) - case _ => None - } - - def asKernelNotification(offset : Option[v2_3.InputPosition])(notification: InternalNotification): org.neo4j.graphdb.Notification = notification match { + def asKernelNotification(offset : Option[v2_3.InputPosition])(notification: InternalNotification): org.neo4j.graphdb.Notification = notification match { case CartesianProductNotification(pos, variables) => NotificationCode.CARTESIAN_PRODUCT.notification(pos.withOffset(offset).asInputPosition, NotificationDetail.Factory.cartesianProduct(variables.asJava)) case LegacyPlannerNotification => @@ -100,16 +94,15 @@ object ExecutionResultWrapper { } -class ExecutionResultWrapper(val inner: InternalExecutionResult, val planner: PlannerName, val runtime: RuntimeName, +class ExecutionResultWrapper(val inner: InternalExecutionResult, + val planner: PlannerName, + val runtime: RuntimeName, val preParsingNotifications: Set[org.neo4j.graphdb.Notification], val offset : Option[v2_3.InputPosition]) extends internal.runtime.InternalExecutionResult { - override def planDescriptionRequested: Boolean = inner.planDescriptionRequested - - override def javaIterator: ResourceIterator[util.Map[String, Any]] = inner.javaIterator - - override def columnAs[T](column: String): Iterator[Nothing] = inner.columnAs(column) + override def javaIterator: ResourceIterator[util.Map[String, AnyRef]] = + inner.javaIterator.asInstanceOf[ResourceIterator[util.Map[String, AnyRef]]] def queryStatistics(): QueryStatistics = { val i = inner.queryStatistics() @@ -197,10 +190,6 @@ class ExecutionResultWrapper(val inner: InternalExecutionResult, val planner: Pl PlanDescriptionImpl(Id.INVALID_ID, name, children, arguments, planDescription.identifiers) } - override def hasNext: Boolean = inner.hasNext - - override def next(): Map[String, Any] = inner.next() - override def close(): Unit = inner.close() def queryType: InternalQueryType = inner.executionType.queryType() match { @@ -221,9 +210,6 @@ class ExecutionResultWrapper(val inner: InternalExecutionResult, val planner: Pl else internal.runtime.NormalMode } - override def withNotifications(notification: Notification*): internal.runtime.InternalExecutionResult = - new ExecutionResultWrapper(inner, planner, runtime, preParsingNotifications ++ notification, offset) - override def fieldNames(): Array[String] = inner.columns.toArray override def accept[E <: Exception](visitor: QueryResult.QueryResultVisitor[E]): Unit = diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_1/Cypher31Compiler.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_1/Cypher31Compiler.scala index 26c08114841a6..9bb5c60cae968 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_1/Cypher31Compiler.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_1/Cypher31Compiler.scala @@ -25,8 +25,9 @@ import org.neo4j.cypher.CypherExecutionMode import org.neo4j.cypher.internal._ import org.neo4j.cypher.internal.compatibility._ import org.neo4j.cypher.internal.compatibility.v3_1.helpers._ +import org.neo4j.cypher.internal.compatibility.v3_5.runtime.helpers.InternalWrapping.{asKernelNotification => asKernelNotification3_5} import org.neo4j.cypher.internal.compiler.v3_1 -import org.neo4j.cypher.internal.compiler.v3_1.executionplan.{InternalExecutionResult, ExecutionPlan => ExecutionPlan_v3_1} +import org.neo4j.cypher.internal.compiler.v3_1.executionplan.{ExecutionPlan => ExecutionPlan_v3_1, InternalExecutionResult => InternalExecutionResult3_1} import org.neo4j.cypher.internal.compiler.v3_1.tracing.rewriters.RewriterStepSequencer import org.neo4j.cypher.internal.compiler.v3_1.{InfoLogger, ExplainMode => ExplainModev3_1, NormalMode => NormalModev3_1, ProfileMode => ProfileModev3_1, _} import org.neo4j.cypher.internal.frontend.v3_1.{InputPosition => InputPosition3_1} @@ -43,7 +44,8 @@ import org.neo4j.kernel.monitoring.{Monitors => KernelMonitors} import org.neo4j.logging.Log import org.neo4j.values.AnyValue import org.neo4j.values.virtual.MapValue -import org.opencypher.v9_0.{frontend => v3_5} +import org.opencypher.v9_0.frontend.phases +import org.opencypher.v9_0.util.InternalNotification import scala.collection.mutable @@ -89,12 +91,18 @@ trait Cypher31Compiler extends CachingPlanner[PreparedQuerySyntax] with Compiler } exceptionHandler.runSafely { val innerParams = typeConversions.asPrivateMap(params) - val innerResult: InternalExecutionResult = inner - .run(queryContext(transactionalContext), innerExecutionMode, innerParams) - new ExecutionResult( - new ClosingExecutionResult( + val innerResult: InternalExecutionResult3_1 = + inner.run(queryContext(transactionalContext), innerExecutionMode, innerParams) + new ExecutionResult( // javacompat + new ClosingExecutionResult( // closing transactionalContext.executingQuery(), - new ExecutionResultWrapper(innerResult, inner.plannerUsed, inner.runtimeUsed, preParsingNotifications, Some(offSet)), + new ExecutionResultWrapper( // 3.5 wrapping + innerResult, // 3.1 + inner.plannerUsed, + inner.runtimeUsed, + preParsingNotifications, + Some(offSet) + ), exceptionHandler.runSafely) ) } @@ -123,8 +131,8 @@ trait Cypher31Compiler extends CachingPlanner[PreparedQuerySyntax] with Compiler } override def compile(preParsedQuery: PreParsedQuery, - tracer: v3_5.phases.CompilationPhaseTracer, - preParsingNotifications: Set[org.neo4j.graphdb.Notification], + tracer: phases.CompilationPhaseTracer, + preParsingNotifications: Set[InternalNotification], transactionalContext: TransactionalContext, params: MapValue ): ExecutableQuery = { @@ -147,7 +155,7 @@ trait Cypher31Compiler extends CachingPlanner[PreparedQuerySyntax] with Compiler new Cypher31ExecutableQuery( executionPlan3_1, - preParsingNotifications, + preParsingNotifications.map(asKernelNotification3_5(None)), position3_1, Seq.empty[String], ValueConversion.asValues(extractedParameters)) diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_1/ExecutionResultWrapper.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_1/ExecutionResultWrapper.scala index 58cc26627f933..9f5dac1d59df2 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_1/ExecutionResultWrapper.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_1/ExecutionResultWrapper.scala @@ -24,10 +24,9 @@ import java.util import org.neo4j.cypher.internal import org.neo4j.cypher.internal._ -import org.neo4j.cypher.internal.compatibility._ import org.neo4j.cypher.internal.compatibility.v3_1.ExecutionResultWrapper.asKernelNotification import org.neo4j.cypher.internal.compiler.v3_1 -import org.neo4j.cypher.internal.compiler.v3_1.executionplan.{InternalExecutionResult, _} +import org.neo4j.cypher.internal.compiler.v3_1.executionplan.{InternalExecutionResult => InternalExecutionResult3_1, _} import org.neo4j.cypher.internal.compiler.v3_1.spi.{InternalResultRow, InternalResultVisitor} import org.neo4j.cypher.internal.compiler.v3_1.{PlannerName, ExplainMode => ExplainModev3_1, NormalMode => NormalModev3_1, ProfileMode => ProfileModev3_1, _} import org.neo4j.cypher.internal.frontend.v3_1.notification.{DeprecatedPlannerNotification, InternalNotification, PlannerUnsupportedNotification, RuntimeUnsupportedNotification, _} @@ -35,10 +34,7 @@ import org.neo4j.cypher.internal.frontend.v3_1.{SemanticDirection => SemanticDir import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription.Arguments import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription.Arguments._ import org.neo4j.cypher.internal.runtime.planDescription.{Argument, Children, NoChildren, PlanDescriptionImpl, SingleChild, TwoChildren, InternalPlanDescription => InternalPlanDescription3_4} -import org.neo4j.cypher.internal.runtime.{ExplainMode, NormalMode, ProfileMode, QueryStatistics} -import org.opencypher.v9_0.util.attribution.Id -import org.opencypher.v9_0.util.{symbols => symbolsv3_5} -import org.opencypher.v9_0.expressions.SemanticDirection.{BOTH, INCOMING, OUTGOING} +import org.neo4j.cypher.internal.runtime.{ExplainMode, NormalMode, ProfileMode, QueryStatistics, InternalExecutionResult => InternalExecutionResult3_5} import org.neo4j.cypher.internal.v3_5.logical.plans.QualifiedName import org.neo4j.cypher.result.QueryResult import org.neo4j.cypher.result.QueryResult.Record @@ -48,17 +44,21 @@ import org.neo4j.graphdb.impl.notification.{NotificationCode, NotificationDetail import org.neo4j.graphdb.{InputPosition, Notification, ResourceIterator} import org.neo4j.kernel.impl.util.ValueUtils import org.neo4j.values.AnyValue +import org.opencypher.v9_0.expressions.SemanticDirection.{BOTH, INCOMING, OUTGOING} +import org.opencypher.v9_0.util.attribution.Id +import org.opencypher.v9_0.util.{symbols => symbolsv3_5} import scala.collection.JavaConverters._ -class ExecutionResultWrapper(val inner: InternalExecutionResult, val planner: PlannerName, val runtime: RuntimeName, +class ExecutionResultWrapper(val inner: InternalExecutionResult3_1, + val planner: PlannerName, + val runtime: RuntimeName, val preParsingNotification: Set[org.neo4j.graphdb.Notification], val offset : Option[frontend.v3_1.InputPosition]) - extends internal.runtime.InternalExecutionResult { + extends InternalExecutionResult3_5 { - override def planDescriptionRequested: Boolean = inner.planDescriptionRequested - override def javaIterator: ResourceIterator[util.Map[String, Any]] = inner.javaIterator - override def columnAs[T](column: String): Iterator[Nothing] = inner.columnAs(column) + override def javaIterator: ResourceIterator[util.Map[String, AnyRef]] = + inner.javaIterator.asInstanceOf[ResourceIterator[util.Map[String, AnyRef]]] override def queryStatistics(): QueryStatistics = { val i = inner.queryStatistics() @@ -165,8 +165,6 @@ class ExecutionResultWrapper(val inner: InternalExecutionResult, val planner: Pl case symbols3_1.ListType(t) => symbolsv3_5.ListType(lift(t)) } - override def hasNext: Boolean = inner.hasNext - override def next(): Map[String, Any] = inner.next() override def close(): Unit = inner.close() override def queryType: internal.runtime.InternalQueryType = inner.executionType match { @@ -201,9 +199,6 @@ class ExecutionResultWrapper(val inner: InternalExecutionResult, val planner: Pl case NormalModev3_1 => NormalMode } - override def withNotifications(notification: Notification*): internal.runtime.InternalExecutionResult = - new ExecutionResultWrapper(inner, planner, runtime, preParsingNotification ++ notification, offset) - override def fieldNames(): Array[String] = inner.columns.toArray override def accept[E <: Exception](visitor: QueryResult.QueryResultVisitor[E]): Unit = @@ -215,12 +210,6 @@ class ExecutionResultWrapper(val inner: InternalExecutionResult, val planner: Pl } object ExecutionResultWrapper { - def unapply(v: Any): Option[(InternalExecutionResult, PlannerName, RuntimeName, Set[org.neo4j.graphdb.Notification], Option[frontend.v3_1.InputPosition])] = v match { - case closing: ClosingExecutionResult => unapply(closing.inner) - case wrapper: ExecutionResultWrapper => Some((wrapper.inner, wrapper.planner, wrapper.runtime, wrapper.preParsingNotification, wrapper.offset)) - case _ => None - } - def asKernelNotification(offset : Option[frontend.v3_1.InputPosition])(notification: InternalNotification): org.neo4j.graphdb.Notification = notification match { case CartesianProductNotification(pos, variables) => NotificationCode.CARTESIAN_PRODUCT.notification(pos.withOffset(offset).asInputPosition, NotificationDetail.Factory.cartesianProduct(variables.asJava)) diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_1/typeConversions.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_1/typeConversions.scala index b59e892518ae1..5038cf1429be8 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_1/typeConversions.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_1/typeConversions.scala @@ -19,23 +19,22 @@ */ package org.neo4j.cypher.internal.compatibility.v3_1 -import java.util.Collections - import org.neo4j.cypher.internal.compiler.v3_1.helpers.RuntimeTypeConverter import org.neo4j.cypher.internal.compiler.v3_1.{CRS, Coordinate, Geometry, Point} import org.neo4j.cypher.internal.frontend.v3_1.helpers.Eagerly import org.neo4j.graphdb.spatial +import org.neo4j.values.storable.{CoordinateReferenceSystem, Values} import scala.collection.JavaConverters._ object typeConversions extends RuntimeTypeConverter { - override def asPublicType = { + override def asPublicType: Any => Any = { case point: Point => asPublicPoint(point) - case geometry: Geometry => asPublicGeometry(geometry) + case geometry: Geometry => throw new IllegalStateException("There are no non-point geometries in 3.1") case other => other } - override def asPrivateType = { + override def asPrivateType: Any => Any = { case map: Map[_, _] => asPrivateMap(map.asInstanceOf[Map[String, Any]]) case seq: Seq[_] => seq.map(asPrivateType) case javaMap: java.util.Map[_, _] => Eagerly.immutableMapValues(javaMap.asScala, asPrivateType) @@ -46,32 +45,8 @@ object typeConversions extends RuntimeTypeConverter { case value => value } - private def asPublicPoint(point: Point) = new spatial.Point { - override def getGeometryType = "Point" - - override def getCRS: spatial.CRS = asPublicCRS(point.crs) - - override def getCoordinates: java.util.List[spatial.Coordinate] = Collections - .singletonList(new spatial.Coordinate(point.coordinate.values: _*)) - } - - private def asPublicGeometry(geometry: Geometry) = new spatial.Geometry { - override def getGeometryType: String = geometry.geometryType - - override def getCRS: spatial.CRS = asPublicCRS(geometry.crs) - - override def getCoordinates = geometry.coordinates.map { c => - new spatial.Coordinate(c.values: _*) - }.toIndexedSeq.asJava - } - - private def asPublicCRS(crs: CRS) = new spatial.CRS { - override def getType: String = crs.name - - override def getHref: String = crs.url - - override def getCode: Int = crs.code - } + private def asPublicPoint(point: Point): org.neo4j.graphdb.spatial.Point = + Values.pointValue(CoordinateReferenceSystem.get(point.crs.url), point.coordinate.values:_*) def asPrivateMap(incoming: Map[String, Any]): Map[String, Any] = Eagerly.immutableMapValues[String,Any, Any](incoming, asPrivateType) diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_4/Cypher34Planner.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_4/Cypher34Planner.scala index 0fe8f13e2bc8c..0e8641aff718f 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_4/Cypher34Planner.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_4/Cypher34Planner.scala @@ -46,14 +46,13 @@ import org.neo4j.cypher.internal.spi.v3_4.{ExceptionTranslatingPlanContext => Ex import org.neo4j.cypher.internal.util.{v3_4 => utilV3_4} import org.neo4j.cypher.internal.v3_4.expressions.{Expression, Parameter} import org.neo4j.cypher.{CypherPlannerOption, CypherUpdateStrategy, CypherVersion} -import org.neo4j.graphdb.Notification import org.neo4j.helpers.collection.Pair import org.neo4j.kernel.impl.query.TransactionalContext import org.neo4j.kernel.monitoring.{Monitors => KernelMonitors} import org.neo4j.logging.Log import org.neo4j.values.virtual.MapValue import org.opencypher.v9_0.frontend.PlannerName -import org.opencypher.v9_0.frontend.phases.{CompilationPhaseTracer, RecordingNotificationLogger => RecordingNotificationLoggerv3_5} +import org.opencypher.v9_0.frontend.phases.{CompilationPhaseTracer, InternalNotificationLogger => InternalNotificationLoggerv3_5} import org.opencypher.v9_0.util.attribution.SequentialIdGen case class Cypher34Planner(configv3_5: CypherPlannerConfiguration, @@ -117,7 +116,7 @@ case class Cypher34Planner(configv3_5: CypherPlannerConfiguration, override def parseAndPlan(preParsedQuery: PreParsedQuery, tracer: CompilationPhaseTracer, - preParsingNotifications: Set[Notification], + notificationLoggerv3_5: InternalNotificationLoggerv3_5, transactionalContext: TransactionalContext, params: MapValue ): LogicalPlanResult = { @@ -125,7 +124,6 @@ case class Cypher34Planner(configv3_5: CypherPlannerConfiguration, val inputPositionV3_4 = as3_4(preParsedQuery.offset) val inputPositionv3_5 = preParsedQuery.offset val notificationLoggerV3_4 = new RecordingNotificationLoggerV3_4(Some(inputPositionV3_4)) - val notificationLoggerv3_5 = new RecordingNotificationLoggerv3_5(Some(inputPositionv3_5)) runSafely { val syntacticQuery = diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/Cypher35Planner.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/Cypher35Planner.scala index 76a5ba9148e8c..97ea36335a1fd 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/Cypher35Planner.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/Cypher35Planner.scala @@ -108,13 +108,11 @@ case class Cypher35Planner(config: CypherPlannerConfiguration, override def parseAndPlan(preParsedQuery: PreParsedQuery, tracer: CompilationPhaseTracer, - preParsingNotifications: Set[Notification], + notificationLogger: InternalNotificationLogger, transactionalContext: TransactionalContext, params: MapValue ): LogicalPlanResult = { - val notificationLogger = new RecordingNotificationLogger(Some(preParsedQuery.offset)) - runSafely { val syntacticQuery = getOrParse(preParsedQuery, new Parser3_5(planner, notificationLogger, preParsedQuery.offset, tracer)) @@ -146,7 +144,7 @@ case class Cypher35Planner(config: CypherPlannerConfiguration, checkForSchemaChanges(planContext) - // If the query is not cached we want to do the full planning + creating executable plan + // If the query is not cached we want to do the full planning def createPlan(): CacheableLogicalPlan = { val logicalPlanState = planner.planPreparedQuery(preparedQuery, context) notification.LogicalPlanNotifications @@ -184,7 +182,7 @@ case class Cypher35Planner(config: CypherPlannerConfiguration, } private[v3_5] class Parser3_5(planner: v3_5.CypherPlanner[PlannerContext], - notificationLogger: RecordingNotificationLogger, + notificationLogger: InternalNotificationLogger, offset: InputPosition, tracer: CompilationPhaseTracer ) extends Parser[BaseState] { diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/ExceptionTranslatingQueryContext.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/ExceptionTranslatingQueryContext.scala index 12e575d0ddd6a..2bfd71f270661 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/ExceptionTranslatingQueryContext.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/ExceptionTranslatingQueryContext.scala @@ -199,9 +199,6 @@ class ExceptionTranslatingQueryContext(val inner: QueryContext) extends QueryCon allowed: Array[String]): UserDefinedAggregator = translateException(inner.aggregateFunction(name, allowed)) - override def isGraphKernelResultValue(v: Any): Boolean = - translateException(inner.isGraphKernelResultValue(v)) - override def withAnyOpenQueryContext[T](work: (QueryContext) => T): T = inner.withAnyOpenQueryContext(qc => translateException( @@ -265,7 +262,7 @@ class ExceptionTranslatingQueryContext(val inner: QueryContext) extends QueryCon override def nodeIsDense(node: Long) = translateException(inner.nodeIsDense(node)) - override def asObject(value: AnyValue) = + override def asObject(value: AnyValue): AnyRef = translateException(inner.asObject(value)) override def variableLengthPathExpand(realNode: Long, minHops: Option[Int], maxHops: Option[Int], direction: SemanticDirection, relTypes: Seq[String]) = diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/ExecutionResultDumper.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/ExecutionResultDumper.scala deleted file mode 100644 index c5fdd39931316..0000000000000 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/ExecutionResultDumper.scala +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright (c) 2002-2018 "Neo4j," - * Neo4j Sweden AB [http://neo4j.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.cypher.internal.compatibility.v3_5.runtime - -import java.io.{PrintWriter, StringWriter} - -import org.neo4j.cypher.internal.compiler.v3_5.helpers.IsList -import org.neo4j.cypher.internal.runtime.interpreted.commands.values.KeyToken -import org.neo4j.cypher.internal.runtime.{QueryContext, QueryStatistics} -import org.neo4j.graphdb.{Node, PropertyContainer, Relationship} - -import scala.collection.Map - -case class ExecutionResultDumper(result: Seq[Map[String, Any]], columns: List[String], queryStatistics: QueryStatistics) { - - def dumpToString(implicit query: QueryContext): String = { - val stringWriter = new StringWriter() - val writer = new PrintWriter(stringWriter) - dumpToString(writer) - writer.close() - stringWriter.getBuffer.toString - } - - def dumpToString(writer: PrintWriter)(implicit query: QueryContext) { - if (columns.nonEmpty) { - val headers = columns.map((c) => Map[String, Any](c -> Some(c))).reduceLeft(_ ++ _) - val columnSizes = calculateColumnSizes - val headerLine = createString(columnSizes, headers) - val lineWidth = headerLine.length - 2 - val --- = "+" + repeat("-", lineWidth) + "+" - - val row = if (result.size > 1) "rows" else "row" - val footer = "%d %s".format(result.size, row) - - writer.println(---) - writer.println(headerLine) - writer.println(---) - - result.foreach(resultLine => writer.println(createString(columnSizes, resultLine))) - - writer.println(---) - writer.println(footer) - - if (queryStatistics.containsUpdates) { - writer.print(queryStatistics.toString) - } - } else { - if (queryStatistics.containsUpdates) { - writer.println("+-------------------+") - writer.println("| No data returned. |") - writer.println("+-------------------+") - writer.print(queryStatistics.toString) - } else { - writer.println("+--------------------------------------------+") - writer.println("| No data returned, and nothing was changed. |") - writer.println("+--------------------------------------------+") - } - } - } - - def createString(columnSizes: Map[String, Int], m: Map[String, Any])(implicit query: QueryContext): String = { - columns.map(c => { - val length = columnSizes.get(c).get - val txt = serialize(m.get(c).get, query) - val value = makeSize(txt, length) - value - }).mkString("| ", " | ", " |") - } - - def calculateColumnSizes(implicit query: QueryContext): Map[String, Int] = { - val columnSizes = new scala.collection.mutable.OpenHashMap[String, Int] ++ columns.map(name => name -> name.size) - - result.foreach((m) => { - m.foreach((kv) => { - val length = serialize(kv._2, query).size - if (!columnSizes.contains(kv._1) || columnSizes.get(kv._1).get < length) { - columnSizes.put(kv._1, length) - } - }) - }) - columnSizes.toMap - } - - private def makeSize(txt: String, wantedSize: Int): String = { - val actualSize = txt.length() - if (actualSize > wantedSize) { - txt.slice(0, wantedSize) - } else if (actualSize < wantedSize) { - txt + repeat(" ", wantedSize - actualSize) - } else txt - } - - private def repeat(x: String, size: Int): String = (1 to size).map((i) => x).mkString - - private def serializeProperties(x: PropertyContainer, qtx: QueryContext): String = { - val (ops, id, deleted) = x match { - case n: Node => (qtx.nodeOps, n.getId, qtx.nodeOps.isDeletedInThisTx(n.getId)) - case r: Relationship => (qtx.relationshipOps, r.getId, qtx.relationshipOps.isDeletedInThisTx(r.getId)) - } - - val keyValStrings = if (deleted) Array("deleted") - else ops.propertyKeyIds(id). - map(pkId => qtx.getPropertyKeyName(pkId) + ":" + serialize(ops.getProperty(id, pkId).asObject(), qtx)) - - keyValStrings.mkString("{", ",", "}") - } - - import scala.collection.JavaConverters._ - private def serialize(a: Any, qtx: QueryContext): String = a match { - case x: Node => x.toString + serializeProperties(x, qtx) - case x: Relationship => ":" + x.getType.name() + "[" + x.getId + "]" + serializeProperties(x, qtx) - case x: Any if x.isInstanceOf[Map[_, _]] => makeString(_ => x.asInstanceOf[Map[String, Any]], qtx) - case x: Any if x.isInstanceOf[java.util.Map[_, _]] => makeString(_ => x.asInstanceOf[java.util.Map[String, Any]].asScala, qtx) - case IsList(coll) => coll.map(elem => serialize(elem, qtx)).mkString("[", ",", "]") - case x: String => "\"" + x + "\"" - case v: KeyToken => v.name - case Some(x) => x.toString - case null => "" - case x => x.toString - } - - private def makeString(m: QueryContext => Map[String, Any], qtx: QueryContext) = m(qtx).map { - case (k, v) => k + " -> " + serialize(v, qtx) - }.mkString("{", ", ", "}") -} - diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/ExplainExecutionResult.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/ExplainExecutionResult.scala index 1501cf3276b7a..ea9f67ff4b855 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/ExplainExecutionResult.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/ExplainExecutionResult.scala @@ -23,49 +23,40 @@ import java.io.PrintWriter import java.util import java.util.Collections -import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription import org.neo4j.cypher.internal.runtime._ +import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription import org.neo4j.cypher.result.QueryResult.QueryResultVisitor import org.neo4j.graphdb.Result.ResultVisitor import org.neo4j.graphdb.{Notification, ResourceIterator} import org.neo4j.helpers.collection.Iterators case class ExplainExecutionResult(fieldNames: Array[String], - executionPlanDescription: InternalPlanDescription, + planDescription: InternalPlanDescription, queryType: InternalQueryType, notifications: Set[Notification]) extends InternalExecutionResult { - def javaIterator: ResourceIterator[util.Map[String, Any]] = Iterators.emptyResourceIterator() - def columnAs[T](column: String) = Iterator.empty + override def javaIterator: ResourceIterator[util.Map[String, AnyRef]] = Iterators.emptyResourceIterator() override def javaColumns: util.List[String] = Collections.emptyList() - def queryStatistics() = QueryStatistics() + override def queryStatistics() = QueryStatistics() - def dumpToString(writer: PrintWriter) { - writer.print(dumpToString) - } + override def dumpToString(writer: PrintWriter): Unit = writer.print(dumpToString) - val dumpToString: String = + override val dumpToString: String = """+--------------------------------------------+ || No data returned, and nothing was changed. | |+--------------------------------------------+ |""".stripMargin - def javaColumnAs[T](column: String): ResourceIterator[T] = Iterators.emptyResourceIterator() - - def planDescriptionRequested = true - - def close() {} - - def next() = Iterator.empty.next() + override def javaColumnAs[T](column: String): ResourceIterator[T] = Iterators.emptyResourceIterator() - def hasNext = false + override def close(): Unit = {} override def accept[EX <: Exception](visitor: ResultVisitor[EX]): Unit = {} override def accept[EX <: Exception](visitor: QueryResultVisitor[EX]): Unit = {} override def executionMode: ExecutionMode = ExplainMode - override def withNotifications(added: Notification*): InternalExecutionResult = copy(notifications = notifications ++ added) + override def executionPlanDescription(): InternalPlanDescription = planDescription } diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/Completable.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/IteratorBasedResult.scala similarity index 68% rename from community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/Completable.scala rename to community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/IteratorBasedResult.scala index 3754a4f10d2dd..33affb5cb8d7c 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/Completable.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/IteratorBasedResult.scala @@ -17,12 +17,10 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -package org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan +package org.neo4j.cypher.internal.compatibility.v3_5.runtime -/** - * This trait is used to signal up the abstraction levels that running a query is completed, either - * because the results have been exhausted, or because a failure has terminated the execution of the query. - */ -trait Completable { - def completed(success: Boolean): Unit -} +import org.neo4j.cypher.result.QueryResult +import org.neo4j.values.AnyValue + +case class IteratorBasedResult(mapIterator: Iterator[collection.Map[String, AnyValue]], + recordIterator: Option[Iterator[QueryResult.Record]] = None) diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/PipeExecutionResult.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/PipeExecutionResult.scala index a41a28be29a44..578ef2e4db61d 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/PipeExecutionResult.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/PipeExecutionResult.scala @@ -19,115 +19,55 @@ */ package org.neo4j.cypher.internal.compatibility.v3_5.runtime -import java.io.PrintWriter import java.util -import org.neo4j.cypher.CypherVersion -import org.neo4j.cypher.internal.compatibility.v3_5.runtime.helpers.MapBasedRow -import org.opencypher.v9_0.util.Eagerly.immutableMapValues import org.neo4j.cypher.internal.runtime._ import org.neo4j.cypher.internal.runtime.interpreted.pipes.QueryState -import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription -import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription.Arguments.{RuntimeVersion, Version} -import org.neo4j.cypher.result.QueryResult import org.neo4j.cypher.result.QueryResult.QueryResultVisitor -import org.neo4j.graphdb.Result.ResultVisitor -import org.neo4j.graphdb.{NotFoundException, Notification, ResourceIterator} +import org.neo4j.cypher.result.{QueryProfile, RuntimeResult} +import org.neo4j.graphdb.ResourceIterator +import org.neo4j.values.AnyValue -import scala.collection.JavaConverters._ -import scala.collection.{Map, mutable} - -class PipeExecutionResult(val result: ResultIterator, +class PipeExecutionResult(val result: IteratorBasedResult, val fieldNames: Array[String], val state: QueryState, - val executionPlanBuilder: () => InternalPlanDescription, - val executionMode: ExecutionMode, - val queryType: InternalQueryType) - extends InternalExecutionResult { + override val queryProfile: QueryProfile) + extends RuntimeResult { self => private val query = state.query - val javaValues = new RuntimeJavaValueConverter(query.isGraphKernelResultValue) - val scalaValues = new RuntimeScalaValueConverter(query.isGraphKernelResultValue) - lazy val dumpToString = withDumper(dumper => dumper.dumpToString(_)) - - def dumpToString(writer: PrintWriter) { withDumper(dumper => dumper.dumpToString(writer)(_)) } - - def executionPlanDescription(): InternalPlanDescription = - executionPlanBuilder() - .addArgument(RuntimeVersion(CypherVersion.default.name)) - - def javaColumnAs[T](column: String): ResourceIterator[T] = new WrappingResourceIterator[T] { - def hasNext = self.hasNext - def next() = query.asObject(result.next().getOrElse(column, columnNotFoundException(column, columns))).asInstanceOf[T] - } - - def columnAs[T](column: String): Iterator[T] = - if (this.columns.contains(column)) map { case m => getAnyColumn(column, m).asInstanceOf[T] } - else columnNotFoundException(column, columns) - - def javaIterator: ResourceIterator[java.util.Map[String, Any]] = new WrappingResourceIterator[util.Map[String, Any]] { - def hasNext = self.hasNext - def next() = immutableMapValues(result.next(), query.asObject).asJava + val javaValues = new RuntimeJavaValueConverter(isGraphKernelResultValue) + def isIterable: Boolean = true + + def asIterator: ResourceIterator[java.util.Map[String, AnyRef]] = + new WrappingResourceIterator[util.Map[String, AnyRef]] { + private val inner = result.mapIterator + def hasNext: Boolean = inner.hasNext + def next(): util.Map[String, AnyRef] = { + val scalaRow: collection.Map[String, AnyValue] = inner.next() + val javaRow = new util.HashMap[String, AnyRef](scalaRow.size) + for (kv <- scalaRow) + javaRow.put(kv._1, query.asObject(kv._2)) + javaRow + } } - override def toList: List[Predef.Map[String, Any]] = result.toList.map(immutableMapValues(_, query.asObject)) - .map(immutableMapValues(_, scalaValues.asDeepScalaValue)) - - def hasNext = result.hasNext - - def next() = immutableMapValues(immutableMapValues(result.next(), query.asObject), scalaValues.asDeepScalaValue) - - def queryStatistics() = state.getStatistics + override def queryStatistics(): QueryStatistics = state.getStatistics - def close() { result.close() } - - def planDescriptionRequested = executionMode == ExplainMode || executionMode == ProfileMode - - private def columnNotFoundException(column: String, expected: Iterable[String]) = - throw new NotFoundException("No column named '" + column + "' was found. Found: " + expected.mkString("(\"", "\", \"", "\")")) - - private def getAnyColumn[T](column: String, m: Map[String, Any]): Any = - m.getOrElse(column, columnNotFoundException(column, m.keys)) - - - private def withDumper[T](f: (ExecutionResultDumper) => (QueryContext => T)): T = { - val result = toList - state.query.withAnyOpenQueryContext( qtx => f(ExecutionResultDumper(result, columns, queryStatistics()))(qtx) ) - } + override def close(): Unit = {} private trait WrappingResourceIterator[T] extends ResourceIterator[T] { def remove() { throw new UnsupportedOperationException("remove") } def close() { self.close() } } - //notifications only present for EXPLAIN - override val notifications = Iterable.empty[Notification] - override def withNotifications(notification: Notification*): InternalExecutionResult = this - - def accept[EX <: Exception](visitor: QueryResultVisitor[EX]): Unit = { - try { - val maybeRecordIterator = result.recordIterator - if (maybeRecordIterator.isDefined) - javaValues.feedQueryResultRecordIteratorToVisitable(maybeRecordIterator.get).accept(visitor) - else - javaValues.feedIteratorToVisitable(result.map(r => fieldNames.map(r))).accept(visitor) - } finally { - self.close() - } - } - - override def accept[E <: Exception](visitor: ResultVisitor[E]): Unit = { - accept(new QueryResultVisitor[E] { - override def visit(record: QueryResult.Record): Boolean = { - val fields = record.fields() - val mapData = new mutable.AnyRefMap[String, Any](fieldNames.length) - for (i <- 0 until fieldNames.length) { - mapData.put(fieldNames(i), state.query.asObject(fields(i))) - } - visitor.visit(new MapBasedRow(mapData)) - } - }) + override def accept[EX <: Exception](visitor: QueryResultVisitor[EX]): Unit = { + val maybeRecordIterator = result.recordIterator + if (maybeRecordIterator.isDefined) + javaValues.feedQueryResultRecordIteratorToVisitable(maybeRecordIterator.get).accept(visitor) + else + javaValues.feedIteratorToVisitable(result.mapIterator.map(r => fieldNames.map(r))).accept(visitor) } + override def isExhausted: Boolean = !result.mapIterator.hasNext } diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/ResultIterator.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/ResultIterator.scala deleted file mode 100644 index 4e1f2024eab5a..0000000000000 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/ResultIterator.scala +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (c) 2002-2018 "Neo4j," - * Neo4j Sweden AB [http://neo4j.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.cypher.internal.compatibility.v3_5.runtime - -import org.opencypher.v9_0.util.{CypherException, TaskCloser} -import org.neo4j.cypher.result.QueryResult -import org.neo4j.values.AnyValue - -trait ResultIterator extends Iterator[collection.Map[String, AnyValue]] { - def toEager: EagerResultIterator - def wasMaterialized: Boolean - def close() - def recordIterator: Option[Iterator[QueryResult.Record]] = None -} - -class EagerResultIterator(result: ResultIterator) extends ResultIterator { - override val toList = result.toList - private val inner = toList.iterator - - override def toEager: EagerResultIterator = this - - override def wasMaterialized = true - - override def hasNext = inner.hasNext - - override def next() = inner.next() - - override def close() = result.close() -} - -class ClosingIterator(inner: Iterator[collection.Map[String, AnyValue]], - closer: TaskCloser, - exceptionDecorator: CypherException => CypherException) extends ResultIterator { - - override def toEager = new EagerResultIterator(this) - - override def wasMaterialized = isEmpty - - override def hasNext: Boolean = { - if (closer.isClosed) false - else { - try { - val innerHasNext = inner.hasNext - if (!innerHasNext) { - close(success = true) - } - innerHasNext - } catch { - case t: Throwable => safeClose(t) - } - } - } - - override def next(): collection.Map[String, AnyValue] = { - if (closer.isClosed) Iterator.empty.next() - try { - inner.next() - } catch { - case t: Throwable => safeClose(t) - } - } - - private def safeClose(t: Throwable) = { - try { - close(success = false) - } catch { - case thrownDuringClose: Throwable => - try { - t.addSuppressed(thrownDuringClose) - } catch { - case _: Throwable => // Ignore - } - } - throw t - } - - override def close() { - close(success = true) - } - - private def close(success: Boolean) = { - try { - closer.close(success) - } catch { - case e: CypherException => - throw exceptionDecorator(e) - } - } -} - -class ClosingQueryResultRecordIterator(inner: Iterator[collection.Map[String, AnyValue]], - closer: TaskCloser, - exceptionDecorator: CypherException => CypherException) - extends ClosingIterator(inner, closer, exceptionDecorator) { - - override def recordIterator = - Some(this.asInstanceOf[Iterator[QueryResult.Record]]) -} diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/DefaultExecutionResultBuilderFactory.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/DefaultExecutionResultBuilderFactory.scala index 44a07c5817dd7..0ee5b52e65b6b 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/DefaultExecutionResultBuilderFactory.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/DefaultExecutionResultBuilderFactory.scala @@ -20,18 +20,14 @@ package org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan import org.neo4j.cypher.internal.compatibility.v3_5.runtime._ -import org.neo4j.cypher.internal.compatibility.v3_5.runtime.helpers.InternalWrapping._ -import org.opencypher.v9_0.frontend.PlannerName -import org.opencypher.v9_0.frontend.phases.InternalNotificationLogger -import org.neo4j.cypher.internal.planner.v3_5.spi.PlanningAttributes.Cardinalities +import org.neo4j.cypher.internal.runtime._ import org.neo4j.cypher.internal.runtime.interpreted.pipes._ import org.neo4j.cypher.internal.runtime.interpreted.{CSVResources, ExecutionContext} -import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription.Arguments.{Runtime, RuntimeImpl} -import org.neo4j.cypher.internal.runtime.planDescription.{InternalPlanDescription, LogicalPlan2PlanDescription} -import org.neo4j.cypher.internal.runtime.{InternalExecutionResult, _} -import org.opencypher.v9_0.util.{CypherException, ProfilerStatisticsNotReadyException, TaskCloser} import org.neo4j.cypher.internal.v3_5.logical.plans.LogicalPlan +import org.neo4j.cypher.result.{QueryProfile, RuntimeResult} import org.neo4j.values.virtual.MapValue +import org.opencypher.v9_0.frontend.phases.InternalNotificationLogger +import org.opencypher.v9_0.util.{CypherException, TaskCloser} import scala.collection.mutable @@ -66,79 +62,38 @@ abstract class BaseExecutionResultBuilderFactory(pipe: Pipe, exceptionDecorator = newDecorator } - override def build(planType: ExecutionMode, - params: MapValue, + override def build(params: MapValue, notificationLogger: InternalNotificationLogger, - plannerName: PlannerName, - runtimeName: RuntimeName, readOnly: Boolean, - cardinalities: Cardinalities): InternalExecutionResult = { + queryProfile: QueryProfile): RuntimeResult = { taskCloser.addTask(queryContext.transactionalContext.close) taskCloser.addTask(queryContext.resources.close) val state = createQueryState(params) try { - createResults(state, planType, notificationLogger, plannerName, runtimeName, readOnly, cardinalities) + createResults(state, notificationLogger, readOnly, queryProfile) } catch { case e: CypherException => taskCloser.close(success = false) throw exceptionDecorator(e) - case (t: Throwable) => + case t: Throwable => taskCloser.close(success = false) throw t } } - private def createResults(state: QueryState, planType: ExecutionMode, + private def createResults(state: QueryState, notificationLogger: InternalNotificationLogger, - plannerName: PlannerName, - runtimeName: RuntimeName, readOnly: Boolean, - cardinalities: Cardinalities): InternalExecutionResult = { - val queryType: InternalQueryType = getQueryType - val planDescription = - () => LogicalPlan2PlanDescription(logicalPlan, plannerName, readOnly, cardinalities) - .addArgument(Runtime(runtimeName.toTextOutput)) - .addArgument(RuntimeImpl(runtimeName.name)) - if (planType == ExplainMode) { - //close all statements - taskCloser.close(success = true) - ExplainExecutionResult(columns.toArray, planDescription(), queryType, - notificationLogger.notifications.map(asKernelNotification(notificationLogger.offset))) - } else { - val results = pipe.createResults(state) - val resultIterator = buildResultIterator(results, readOnly) - val verifyProfileReady = () => { - val isResultReady = resultIterator.wasMaterialized - if (!isResultReady) { - taskCloser.close(success = false) - throw new ProfilerStatisticsNotReadyException() - } - } - val descriptor = buildDescriptor(planDescription, verifyProfileReady) - new PipeExecutionResult(resultIterator, columns.toArray, state, descriptor, planType, queryType) - } + queryProfile: QueryProfile): RuntimeResult = { + val results = pipe.createResults(state) + val resultIterator = buildResultIterator(results, readOnly) + new PipeExecutionResult(resultIterator, columns.toArray, state, queryProfile) } - protected def queryContext = maybeQueryContext.get - - protected def buildResultIterator(results: Iterator[ExecutionContext], readOnly: Boolean): ResultIterator - - private def buildDescriptor(planDescription: () => InternalPlanDescription, verifyProfileReady: () => Unit): () => InternalPlanDescription = - pipeDecorator.decorate(planDescription, verifyProfileReady) - } + protected def queryContext: QueryContext = maybeQueryContext.get - private def getQueryType = { - val queryType = - if (pipe.isInstanceOf[IndexOperationPipe] || pipe.isInstanceOf[ConstraintOperationPipe]) - SCHEMA_WRITE - else if (readOnly) - READ_ONLY - else if (columns.isEmpty) - WRITE - else - READ_WRITE - queryType + protected def buildResultIterator(results: Iterator[ExecutionContext], readOnly: Boolean): IteratorBasedResult } } @@ -148,19 +103,16 @@ case class InterpretedExecutionResultBuilderFactory(pipe: Pipe, logicalPlan: LogicalPlan) extends BaseExecutionResultBuilderFactory(pipe, readOnly, columns, logicalPlan) { - override def create(): ExecutionResultBuilder = - new InterpretedExecutionWorkflowBuilder() + override def create(): ExecutionResultBuilder = InterpretedExecutionWorkflowBuilder() case class InterpretedExecutionWorkflowBuilder() extends BaseExecutionWorkflowBuilder { - override def createQueryState(params: MapValue) = { + override def createQueryState(params: MapValue): QueryState = { new QueryState(queryContext, externalResource, params, pipeDecorator, triadicState = mutable.Map.empty, repeatableReads = mutable.Map.empty) } - override def buildResultIterator(results: Iterator[ExecutionContext], readOnly: Boolean): ResultIterator = { - val closingIterator = new ClosingIterator(results, taskCloser, exceptionDecorator) - val resultIterator = if (!readOnly) closingIterator.toEager else closingIterator - resultIterator + override def buildResultIterator(results: Iterator[ExecutionContext], readOnly: Boolean): IteratorBasedResult = { + IteratorBasedResult(results) } } } diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/ExecutionPlan.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/ExecutionPlan.scala index 54858df6ea322..196753d020348 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/ExecutionPlan.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/ExecutionPlan.scala @@ -20,10 +20,13 @@ package org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan import org.neo4j.cypher.internal.compatibility.v3_5.runtime.RuntimeName -import org.neo4j.cypher.internal.runtime.{ExecutionMode, InternalExecutionResult, QueryContext} +import org.neo4j.cypher.internal.runtime.QueryContext +import org.neo4j.cypher.result.RuntimeResult import org.neo4j.values.virtual.MapValue abstract class ExecutionPlan { - def run(queryContext: QueryContext, planType: ExecutionMode, params: MapValue): InternalExecutionResult + + def run(queryContext: QueryContext, doProfile: Boolean, params: MapValue): RuntimeResult + def runtimeName: RuntimeName } diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/ExecutionResultBuilder.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/ExecutionResultBuilder.scala index 2ac92a85b1765..389443208c2cf 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/ExecutionResultBuilder.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/ExecutionResultBuilder.scala @@ -19,27 +19,22 @@ */ package org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan -import org.opencypher.v9_0.util.CypherException -import org.neo4j.cypher.internal.compatibility.v3_5.runtime.RuntimeName -import org.opencypher.v9_0.frontend.PlannerName -import org.opencypher.v9_0.frontend.phases.InternalNotificationLogger -import org.neo4j.cypher.internal.planner.v3_5.spi.PlanningAttributes.Cardinalities +import org.neo4j.cypher.internal.runtime.QueryContext import org.neo4j.cypher.internal.runtime.interpreted.pipes.PipeDecorator -import org.neo4j.cypher.internal.runtime.{ExecutionMode, InternalExecutionResult, QueryContext} +import org.neo4j.cypher.result.{QueryProfile, RuntimeResult} import org.neo4j.values.virtual.MapValue +import org.opencypher.v9_0.frontend.phases.InternalNotificationLogger +import org.opencypher.v9_0.util.CypherException trait ExecutionResultBuilder { def setQueryContext(context: QueryContext) def setLoadCsvPeriodicCommitObserver(batchRowCount: Long) def setPipeDecorator(newDecorator: PipeDecorator) def setExceptionDecorator(newDecorator: CypherException => CypherException) - def build(planType: ExecutionMode, - params: MapValue, + def build(params: MapValue, notificationLogger: InternalNotificationLogger, - plannerName: PlannerName, - runtimeName: RuntimeName, readOnly: Boolean, - cardinalities: Cardinalities): InternalExecutionResult + queryProfile: QueryProfile): RuntimeResult } trait ExecutionResultBuilderFactory { diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/Provider.java b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/Provider.java index b3553020ec9cb..351b80cc041c2 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/Provider.java +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/Provider.java @@ -22,4 +22,9 @@ public interface Provider { T get(); + + static Provider NULL() + { + return () -> null; + } } diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/StandardInternalExecutionResult.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/StandardInternalExecutionResult.scala index cee2fb6cbc9a8..0dcf7a56ee5c2 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/StandardInternalExecutionResult.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/StandardInternalExecutionResult.scala @@ -24,82 +24,151 @@ import java.util import org.neo4j.cypher.internal.compatibility.v3_5.runtime._ import org.neo4j.cypher.internal.compatibility.v3_5.runtime.helpers.{MapBasedRow, RuntimeTextValueConverter} +import org.neo4j.cypher.internal.compatibility.v3_5.runtime.profiler.PlanDescriptionBuilder import org.neo4j.cypher.internal.runtime._ -import org.opencypher.v9_0.util.{Eagerly, TaskCloser} import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription -import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription.Arguments.{Runtime, RuntimeImpl} -import org.neo4j.cypher.result.QueryResult import org.neo4j.cypher.result.QueryResult.QueryResultVisitor -import org.neo4j.graphdb.{NotFoundException, Notification, ResourceIterator} +import org.neo4j.cypher.result.{QueryResult, RuntimeResult} import org.neo4j.graphdb.Result.{ResultRow, ResultVisitor} +import org.neo4j.graphdb.{NotFoundException, Notification, ResourceIterator} +import org.neo4j.values.AnyValue +import org.opencypher.v9_0.util.{ProfilerStatisticsNotReadyException, TaskCloser} -import scala.collection.{Map, mutable} +import scala.collection.mutable -abstract class StandardInternalExecutionResult(context: QueryContext, runtime: RuntimeName, - taskCloser: Option[TaskCloser] = None) - extends InternalExecutionResult - with Completable { +class StandardInternalExecutionResult(context: QueryContext, + runtime: RuntimeName, + runtimeResult: RuntimeResult, + taskCloser: TaskCloser, + override val queryType: InternalQueryType, + override val notifications: Set[Notification], + override val executionMode: ExecutionMode, + planDescriptionBuilder: PlanDescriptionBuilder) + extends InternalExecutionResult { self => - import scala.collection.JavaConverters._ - - protected val isGraphKernelResultValue = context.isGraphKernelResultValue _ - private val scalaValues = new RuntimeScalaValueConverter(isGraphKernelResultValue) + /* + ======= RESULT MATERIALIZATION ========== + */ - protected def isOpen: Boolean = !isClosed + private var materializedResult: util.ArrayList[Array[AnyValue]] = _ + private def isMaterialized: Boolean = materializedResult != null + private def materializeResult(): Unit = { + materializedResult = new util.ArrayList() + if (isOpen) + try { + runtimeResult.accept(new QueryResultVisitor[Exception] { + override def visit(row: QueryResult.Record): Boolean = { + materializedResult.add(row.fields().clone()) + row.release() + true + } + }) + } catch { + case t: Throwable => + throw closeOnError(t) + } + } - protected def isClosed: Boolean = taskCloser.exists(_.isClosed) + /** + * By policy we materialize the result directly unless it's a read only query. + */ + if (queryType != READ_ONLY) { + materializeResult() + if (queryType == WRITE || queryType == SCHEMA_WRITE) { // .. and if we do not return any rows, we also close all resources + close(true) + } + } - override def hasNext: Boolean = inner.hasNext + /* + ======= OPEN / CLOSE ========== + */ - override def next(): Predef.Map[String, Any] = scalaValues.asDeepScalaMap(inner.next()) + protected def isOpen: Boolean = !isClosed - // Override one of them in subclasses - override def columnAs[T](column: String): Iterator[T] = - if (this.columns.contains(column)) map(m => extractScalaColumn(column, m).asInstanceOf[T]) - else throw columnNotFound(column, columns) + protected def isClosed: Boolean = taskCloser.isClosed - override def javaIterator: ResourceIterator[util.Map[String, Any]] = new ClosingJavaIterator[util.Map[String, Any]] { - override def next(): util.Map[String, Any] = inner.next() + override def close(): Unit = { + close(success = true) } - override def javaColumnAs[T](column: String): ResourceIterator[T] = new ClosingJavaIterator[T] { - override def next(): T = extractJavaColumn(column, inner.next()).asInstanceOf[T] + override def close(success: Boolean): Unit = { + taskCloser.close(success = success) } - override def dumpToString(): String = { - val stringWriter = new StringWriter() - val writer = new PrintWriter(stringWriter) - dumpToString(writer) - writer.close() - stringWriter.getBuffer.toString + /* + ======= CONSUME AS ITERATOR ========== + */ + + override def javaIterator: ResourceIterator[util.Map[String, AnyRef]] = inner + + override def javaColumnAs[T](column: String): ResourceIterator[T] = + new ResourceIterator[T] { + override def hasNext: Boolean = inner.hasNext + override def next(): T = extractJavaColumn(column, inner.next()).asInstanceOf[T] + override def close(): Unit = self.close() + } + + private def extractJavaColumn(column: String, data: util.Map[String, AnyRef]): AnyRef = { + val value = data.get(column) + if (value == null) { + throw new NotFoundException(s"No column named '$column' was found. Found: ${fieldNames().mkString("(\"", "\", \"", "\")")}") + } + value } - override def dumpToString(writer: PrintWriter): Unit = { - val builder = Seq.newBuilder[Map[String, String]] - doInAccept(populateDumpToStringResults(builder)) - formatOutput(writer, columns, builder.result(), queryStatistics()) + protected final lazy val inner: ResourceIterator[util.Map[String, AnyRef]] = { + if (!isMaterialized && runtimeResult.isIterable) + runtimeResult.asIterator() + else { + if (!isMaterialized) + materializeResult() + new MaterializedIterator() + } } - override def planDescriptionRequested: Boolean = executionMode == ExplainMode || executionMode == ProfileMode - override def notifications = Iterable.empty[Notification] + private class MaterializedIterator() extends ResourceIterator[util.Map[String, AnyRef]] { - override def close(): Unit = { - completed(success = true) + private val inner = materializedResult.iterator() + private val columns = fieldNames() + + def hasNext: Boolean = inner.hasNext + + def next(): util.Map[String, AnyRef] = { + val values = inner.next() + val map = new util.HashMap[String, AnyRef]() + for (i <- columns.indices) { + map.put(columns(i), context.asObject(values(i))) + } + map + } + + def remove(): Unit = throw new UnsupportedOperationException("remove") + + def close(): Unit = self.close() } - override def completed(success: Boolean): Unit = { - taskCloser.foreach(_.close(success = success)) + /* + ======= CONSUME WITH VISITOR ========== + */ + + protected def accept(body: ResultRow => Unit): Unit = { + accept(new ResultVisitor[RuntimeException] { + override def visit(row: ResultRow): Boolean = { + body(row) + true + } + }) } override def accept[E <: Exception](visitor: ResultVisitor[E]): Unit = { accept(new QueryResultVisitor[E] { - val names = fieldNames() + private val names = fieldNames() override def visit(record: QueryResult.Record): Boolean = { val fields = record.fields() val mapData = new mutable.AnyRefMap[String, Any](names.length) - for (i <- 0 until names.length) { + for (i <- names.indices) { mapData.put(names(i), context.asObject(fields(i))) } visitor.visit(new MapBasedRow(mapData)) @@ -107,129 +176,74 @@ abstract class StandardInternalExecutionResult(context: QueryContext, runtime: R }) } - /* - * NOTE: This should ony be used for testing, it creates an InternalExecutionResult - * where you can call both toList and dumpToString - */ - def toEagerResultForTestingOnly(): InternalExecutionResult = { - val dumpToStringBuilder = Seq.newBuilder[Map[String, String]] - val result = new util.ArrayList[util.Map[String, Any]]() - if (isOpen) - doInAccept { (row) => - populateResults(result)(row) - populateDumpToStringResults(dumpToStringBuilder)(row) - } - - new StandardInternalExecutionResult(context, runtime, taskCloser) { - - override protected def createInner: util.Iterator[util.Map[String, Any]] = result.iterator() + override def accept[E <: Exception](visitor: QueryResultVisitor[E]): Unit = { - override def executionPlanDescription(): InternalPlanDescription = { - val description = self.executionPlanDescription() - if (!description.arguments.exists(_.isInstanceOf[Runtime])) { - description.addArgument(Runtime(runtime.toTextOutput)) - } - if (!description.arguments.exists(_.isInstanceOf[RuntimeImpl])) { - description.addArgument(RuntimeImpl(runtime.name)) - } - description + if (isMaterialized) { + val rowCursor = new MaterializedResultCursor + while (rowCursor.next()) { + visitor.visit(rowCursor) } - - override def toList: List[Predef.Map[String, Any]] = result.asScala - .map(m => Eagerly.immutableMapValues(m.asScala, scalaValues.asDeepScalaValue)).toList - - override def dumpToString(writer: PrintWriter): Unit = - formatOutput(writer, columns, dumpToStringBuilder.result(), queryStatistics()) - - override def executionMode: ExecutionMode = self.executionMode - - override def queryStatistics(): QueryStatistics = self.queryStatistics() - - override def queryType: InternalQueryType = self.queryType - - override def withNotifications(notification: Notification*): InternalExecutionResult = self - - override def fieldNames(): Array[String] = self.fieldNames() - - override def accept[E <: Exception](visitor: QueryResultVisitor[E]): Unit = self.accept(visitor) - } + close(success = true) + } else if (isOpen) { + runtimeResult.accept(visitor) + close(success = true) + } else + throw new IllegalStateException("Unable to accept visitors after resources have been closed.") } - protected final lazy val inner: util.Iterator[util.Map[String, Any]] = createInner - - protected def createInner: util.Iterator[util.Map[String, Any]] - - protected def doInAccept[T](body: ResultRow => T): Unit = { - if (isOpen) { - accept(new ResultVisitor[RuntimeException] { - override def visit(row: ResultRow): Boolean = { - body(row) - true - } - }) - } else { - throw new IllegalStateException("Unable to accept visitors after resources have been closed.") + class MaterializedResultCursor extends QueryResult.Record { + private var i = -1 + def next(): Boolean = { + i += 1 + i < materializedResult.size() } + override def fields(): Array[AnyValue] = materializedResult.get(i) } - protected def populateDumpToStringResults(builder: mutable.Builder[Map[String, String], Seq[Map[String, String]]]) - (row: ResultRow): builder.type = { - val textValues = new RuntimeTextValueConverter(scalaValues)(context) - val map = new mutable.HashMap[String, String]() - columns.foreach(c => map.put(c, textValues.asTextValue(row.get(c)))) + /* + ======= DUMP TO STRING ========== + */ - builder += map + override def dumpToString(): String = { + val stringWriter = new StringWriter() + val writer = new PrintWriter(stringWriter) + dumpToString(writer) + writer.close() + stringWriter.getBuffer.toString } - protected def populateResults(results: util.List[util.Map[String, Any]])(row: ResultRow): Boolean = { - val map = new util.HashMap[String, Any]() - columns.foreach(c => map.put(c, row.get(c))) - results.add(map) - } + override def dumpToString(writer: PrintWriter): Unit = { + val builder = Seq.newBuilder[Map[String, String]] + val scalaValues = new RuntimeScalaValueConverter(isGraphKernelResultValue) + val runtimeTextValueConverter = new RuntimeTextValueConverter(scalaValues, context.transactionalContext) - private def extractScalaColumn(column: String, data: Map[String, Any]): Any = { - data.getOrElse(column, { - throw columnNotFound(column, data.keys) + accept(row => { + builder += runtimeTextValueConverter.dumpRowToString(fieldNames(), row) }) - } - private def extractJavaColumn(column: String, data: util.Map[String, Any]): Any = { - val value = data.get(column) - if (value == null) { - throw columnNotFound(column, columns) - } - value + formatOutput(writer, runtimeResult.fieldNames(), builder.result(), queryStatistics()) } - private def columnNotFound(column: String, expected: Iterable[String]) = - new NotFoundException(s"No column named '$column' was found. Found: ${expected.mkString("(\"", "\", \"", "\")")}") + /* + ======= META DATA ========== + */ - private abstract class ClosingJavaIterator[A] extends ResourceIterator[A] { + override def queryStatistics(): QueryStatistics = runtimeResult.queryStatistics() - def hasNext: Boolean = self.hasNext + override def fieldNames(): Array[String] = runtimeResult.fieldNames() - def remove() { - throw new UnsupportedOperationException("remove") - } + override lazy val executionPlanDescription: InternalPlanDescription = { - def close() { - self.close() + if (executionMode == ProfileMode) { + if (!runtimeResult.isExhausted) { + taskCloser.close(success = false) + throw new ProfilerStatisticsNotReadyException() + } + planDescriptionBuilder.profile(runtimeResult.queryProfile) + } else { + planDescriptionBuilder.explain() } - } -} - -object StandardInternalExecutionResult { - - // Accept and pull into memory when iterating - trait IterateByAccepting { - self: StandardInternalExecutionResult => - - override def createInner = { - val list = new util.ArrayList[util.Map[String, Any]]() - if (isOpen) doInAccept(populateResults(list)) - list.iterator() - } } } diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/formatOutput.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/formatOutput.scala index 9fc391ef81c94..52d0534133170 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/formatOutput.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/formatOutput.scala @@ -21,16 +21,16 @@ package org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan import java.io.PrintWriter -import org.neo4j.cypher.internal.runtime.QueryStatistics +import org.neo4j.graphdb.QueryStatistics import scala.collection.Map /** * Creates formatted tabular output. */ -object formatOutput extends ((PrintWriter, List[String], Seq[Map[String, String]], QueryStatistics) => Unit) { +object formatOutput extends ((PrintWriter, Seq[String], Seq[Map[String, String]], QueryStatistics) => Unit) { - def apply(writer: PrintWriter, columns: List[String], + def apply(writer: PrintWriter, columns: Seq[String], result: Seq[Map[String, String]], queryStatistics: QueryStatistics) { def makeSize(txt: String, wantedSize: Int): String = { diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/ProcedureCallExecutionPlan.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/ProcedureCallExecutionPlan.scala index c1bcd9b03ec5c..a46ef6769990f 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/ProcedureCallExecutionPlan.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/ProcedureCallExecutionPlan.scala @@ -19,25 +19,19 @@ */ package org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.procs -import org.neo4j.cypher.CypherVersion -import org.neo4j.cypher.internal.{FineToReuse, ReusabilityState} import org.neo4j.cypher.internal.compatibility.v3_5.runtime._ import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.ExecutionPlan -import org.neo4j.cypher.internal.planner.v3_5.spi.{GraphStatistics, ProcedurePlannerName} import org.neo4j.cypher.internal.runtime._ import org.neo4j.cypher.internal.runtime.interpreted.ExecutionContext import org.neo4j.cypher.internal.runtime.interpreted.commands.convert.ExpressionConverters import org.neo4j.cypher.internal.runtime.interpreted.commands.expressions.{Literal, ParameterExpression, Expression => CommandExpression} import org.neo4j.cypher.internal.runtime.interpreted.pipes.{ExternalCSVResource, QueryState} -import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription.Arguments._ -import org.neo4j.cypher.internal.runtime.planDescription.{Argument, NoChildren, PlanDescriptionImpl} -import org.opencypher.v9_0.util.{InvalidArgumentException, TaskCloser} -import org.opencypher.v9_0.util.attribution.Id -import org.opencypher.v9_0.util.symbols.CypherType -import org.opencypher.v9_0.expressions.Expression import org.neo4j.cypher.internal.v3_5.logical.plans.ProcedureSignature -import org.neo4j.graphdb.Notification +import org.neo4j.cypher.result.RuntimeResult import org.neo4j.values.virtual.MapValue +import org.opencypher.v9_0.expressions.Expression +import org.opencypher.v9_0.util.InvalidArgumentException +import org.opencypher.v9_0.util.symbols.CypherType /** * Execution plan for calling procedures @@ -53,7 +47,6 @@ case class ProcedureCallExecutionPlan(signature: ProcedureSignature, argExprs: Seq[Expression], resultSymbols: Seq[(String, CypherType)], resultIndices: Seq[(Int, String)], - notifications: Set[Notification], converter: ExpressionConverters) extends ExecutionPlan { @@ -69,46 +62,10 @@ case class ProcedureCallExecutionPlan(signature: ProcedureSignature, private val maybeDefaultArgs: Seq[Option[CommandExpression]] = signature.inputSignature.map(_.default).map(option => option.map( df => Literal(df.value))) private val zippedArgCandidates = actualArgs.map(Some(_)).zipAll(parameterArgs.zip(maybeDefaultArgs), None, null).map { case (a, (b, c)) => (a, b, c)} - override def run(ctx: QueryContext, planType: ExecutionMode, params: MapValue): InternalExecutionResult = { + override def run(ctx: QueryContext, doProfile: Boolean, params: MapValue): RuntimeResult = { val input = evaluateArguments(ctx, params) - - val taskCloser = new TaskCloser - taskCloser.addTask(ctx.resources.close) - taskCloser.addTask(ctx.transactionalContext.close) - - planType match { - case NormalMode => createNormalExecutionResult(ctx, taskCloser, input, planType) - case ExplainMode => createExplainedExecutionResult(ctx, taskCloser, input, notifications) - case ProfileMode => createProfiledExecutionResult(ctx, taskCloser, input, planType) - } - } - - private def createNormalExecutionResult(ctx: QueryContext, taskCloser: TaskCloser, - input: Seq[Any], planType: ExecutionMode) = { - val descriptionGenerator = () => createNormalPlan val callMode = ProcedureCallMode.fromAccessMode(signature.accessMode) - new ProcedureExecutionResult(ctx, taskCloser, signature.name, signature.id, callMode, input, - resultMappings, descriptionGenerator, planType) - } - - private def createExplainedExecutionResult(ctx: QueryContext, taskCloser: TaskCloser, input: Seq[Any], - notifications: Set[Notification]) = { - // close all statements - taskCloser.close(success = true) - val callMode = ProcedureCallMode.fromAccessMode(signature.accessMode) - val columns = signature.outputSignature.map(_.seq.map(_.name).toList).getOrElse(List.empty) - ExplainExecutionResult(columns.toArray, createNormalPlan, callMode.queryType, notifications) - } - - private def createProfiledExecutionResult(ctx: QueryContext, taskCloser: TaskCloser, - input: Seq[Any], planType: ExecutionMode) = { - val rowCounter = Counter() - val descriptionGenerator = createProfilePlanGenerator(rowCounter) - val callMode = ProcedureCallMode.fromAccessMode(signature.accessMode) - new ProcedureExecutionResult(ctx, taskCloser, signature.name, signature.id, callMode, input, - resultMappings, descriptionGenerator, planType) { - override protected def executeCall: Iterator[Array[AnyRef]] = rowCounter.track(super.executeCall) - } + new ProcedureCallRuntimeResult(ctx, signature.name, signature.id, callMode, input, resultMappings, doProfile) } private def evaluateArguments(ctx: QueryContext, params: MapValue): Seq[Any] = { @@ -127,31 +84,6 @@ case class ProcedureCallExecutionPlan(signature: ProcedureSignature, args.map(expr => ctx.asObject(expr.apply(ExecutionContext.empty, state))) } - private def createNormalPlan = - PlanDescriptionImpl(Id.INVALID_ID, "ProcedureCall", NoChildren, - arguments, - resultSymbols.map(_._1).toSet - ) - - private def createProfilePlanGenerator(rowCounter: Counter) = () => - PlanDescriptionImpl(Id.INVALID_ID, "ProcedureCall", NoChildren, - Seq(createSignatureArgument, DbHits(1), Rows(rowCounter.counted)) ++ arguments, - resultSymbols.map(_._1).toSet - ) - - private def arguments: Seq[Argument] = Seq(createSignatureArgument, - Runtime(ProcedureRuntimeName.toTextOutput), - RuntimeImpl(ProcedureRuntimeName.name), - Planner(ProcedurePlannerName.toTextOutput), - PlannerImpl(ProcedurePlannerName.name), - PlannerVersion(ProcedurePlannerName.version), - Version(s"CYPHER ${CypherVersion.default.name}"), - RuntimeVersion(CypherVersion.default.name)) - - - private def createSignatureArgument: Argument = - Signature(signature.name, Seq.empty, resultSymbols) - override def runtimeName: RuntimeName = ProcedureRuntimeName } diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/ProcedureExecutionResult.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/ProcedureCallRuntimeResult.scala similarity index 59% rename from community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/ProcedureExecutionResult.scala rename to community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/ProcedureCallRuntimeResult.scala index 538bb76e99d89..4d44774be96bd 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/ProcedureExecutionResult.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/ProcedureCallRuntimeResult.scala @@ -23,74 +23,72 @@ import java.time._ import java.time.temporal.TemporalAmount import java.util -import org.neo4j.cypher.internal.compatibility.v3_5.runtime._ -import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.StandardInternalExecutionResult import org.neo4j.cypher.internal.runtime._ -import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription -import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription.Arguments.{Runtime, RuntimeImpl} -import org.opencypher.v9_0.util.symbols.{CypherType, _} -import org.opencypher.v9_0.util.{ProfilerStatisticsNotReadyException, TaskCloser} import org.neo4j.cypher.internal.v3_5.logical.plans.QualifiedName import org.neo4j.cypher.result.QueryResult.{QueryResultVisitor, Record} -import org.neo4j.graphdb.Notification +import org.neo4j.cypher.result.{OperatorProfile, QueryProfile, RuntimeResult} +import org.neo4j.graphdb.ResourceIterator import org.neo4j.graphdb.spatial.{Geometry, Point} import org.neo4j.kernel.impl.util.ValueUtils import org.neo4j.kernel.impl.util.ValueUtils._ import org.neo4j.values.AnyValue -import org.neo4j.values.storable.Values.{of => DONT_USE_OMG, _} +import org.neo4j.values.storable.Values._ import org.neo4j.values.storable._ +import org.opencypher.v9_0.util.symbols.{CypherType, _} /** - * Execution result of a Procedure + * Result of calling a procedure. * * @param context The QueryContext used to communicate with the kernel. - * @param taskCloser called when done with the result, cleans up resources. * @param name The name of the procedure. * @param id The id of the procedure. * @param callMode The call mode of the procedure. * @param args The argument to the procedure. * @param indexResultNameMappings Describes how values at output row indices are mapped onto result columns. - * @param executionPlanDescriptionGenerator Generator for the plan description of the result. - * @param executionMode The execution mode. */ -class ProcedureExecutionResult(context: QueryContext, - taskCloser: TaskCloser, - name: QualifiedName, - id: Option[Int], - callMode: ProcedureCallMode, - args: Seq[Any], - indexResultNameMappings: IndexedSeq[(Int, String, CypherType)], - executionPlanDescriptionGenerator: () => InternalPlanDescription, - val executionMode: ExecutionMode) - extends StandardInternalExecutionResult(context, ProcedureRuntimeName, Some(taskCloser)) { +class ProcedureCallRuntimeResult(context: QueryContext, + name: QualifiedName, + id: Option[Int], + callMode: ProcedureCallMode, + args: Seq[Any], + indexResultNameMappings: IndexedSeq[(Int, String, CypherType)], + profile: Boolean) extends RuntimeResult { + + self => + + private val counter = Counter() override val fieldNames: Array[String] = indexResultNameMappings.map(_._2).toArray - private final val executionResults = executeCall + private final val executionResults: Iterator[Array[AnyRef]] = executeCall // The signature mode is taking care of eagerization - protected def executeCall: Iterator[Array[AnyRef]] = - if (id.nonEmpty) callMode.callProcedure(context, id.get, args) - else callMode.callProcedure(context, name, args) - - override protected def createInner = new util.Iterator[util.Map[String, Any]]() { - override def next(): util.Map[String, Any] = - resultAsMap(executionResults.next()) //TODO!!!¡¡¡¡ - /* - override def next(): util.Map[String, Any] = - try { resultAsMap( executionResults.next( ) ) } - catch { case e: NoSuchElementException => success(); throw e } - - */ - - override def hasNext: Boolean = { - val moreToCome = executionResults.hasNext - if (!moreToCome) { - close() + protected def executeCall: Iterator[Array[AnyRef]] = { + val iterator = + if (id.nonEmpty) callMode.callProcedure(context, id.get, args) + else callMode.callProcedure(context, name, args) + + if (profile) + counter.track(iterator) + else + iterator + } + + override def asIterator(): ResourceIterator[util.Map[String, AnyRef]] = + new ResourceIterator[util.Map[String, AnyRef]]() { + override def next(): util.Map[String, AnyRef] = + resultAsMap(executionResults.next()) + + override def hasNext: Boolean = { + val moreToCome = executionResults.hasNext + if (!moreToCome) { + close() + } + moreToCome } - moreToCome + + override def close(): Unit = self.close() } - } private def transform[T](value: AnyRef, f: T => AnyValue): AnyValue = { if (value == null) NO_VALUE @@ -136,28 +134,32 @@ class ProcedureExecutionResult(context: QueryContext, // of to get accurate stats for procedure code override def queryStatistics(): QueryStatistics = context.getOptStatistics.getOrElse(QueryStatistics()) - override def queryType: InternalQueryType = callMode.queryType - - private def resultAsMap(rowData: Array[AnyRef]): util.Map[String, Any] = { - val mapData = new util.HashMap[String, Any](rowData.length) - indexResultNameMappings.foreach { entry => mapData.put(entry._2, rowData(entry._1)) } - mapData - } - - private def resultAsRefMap(rowData: Array[AnyRef]): util.Map[String, AnyRef] = { + private def resultAsMap(rowData: Array[AnyRef]): util.Map[String, AnyRef] = { val mapData = new util.HashMap[String, AnyRef](rowData.length) indexResultNameMappings.foreach { entry => mapData.put(entry._2, rowData(entry._1)) } mapData } - override def executionPlanDescription(): InternalPlanDescription = executionMode match { - case ProfileMode if executionResults.hasNext => - completed(success = false) - throw new ProfilerStatisticsNotReadyException() - case _ => executionPlanDescriptionGenerator() - .addArgument(Runtime(ProcedureRuntimeName.toTextOutput)) - .addArgument(RuntimeImpl(ProcedureRuntimeName.toTextOutput)) - } + override def isIterable: Boolean = true + + override def isExhausted: Boolean = !executionResults.hasNext + + override def close(): Unit = {} + + override def queryProfile(): QueryProfile = StandaloneProcedureCallProfile(counter.counted) +} + +case class StandaloneProcedureCallProfile(rowCount: Long) extends QueryProfile with OperatorProfile { + + override def operatorProfile(operatorId: Int): OperatorProfile = this + + override def time(): Long = -1 + + override def dbHits(): Long = 1 // for unclear reasons + + override def rows(): Long = rowCount + + override def pageCacheHits(): Long = -1 - override def withNotifications(notification: Notification*): InternalExecutionResult = this + override def pageCacheMisses(): Long = -1 } diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/PureSideEffectExecutionPlan.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/PureSideEffectExecutionPlan.scala deleted file mode 100644 index 170d9906a7f50..0000000000000 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/PureSideEffectExecutionPlan.scala +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2002-2018 "Neo4j," - * Neo4j Sweden AB [http://neo4j.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.procs - -import org.neo4j.cypher.CypherVersion -import org.neo4j.cypher.internal.compatibility.v3_5.runtime._ -import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.ExecutionPlan -import org.neo4j.cypher.internal.planner.v3_5.spi.ProcedurePlannerName -import org.neo4j.cypher.internal.runtime._ -import org.neo4j.cypher.internal.runtime.interpreted.UpdateCountingQueryContext -import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription.Arguments._ -import org.neo4j.cypher.internal.runtime.planDescription.{NoChildren, PlanDescriptionImpl} -import org.neo4j.values.virtual.MapValue -import org.opencypher.v9_0.util.attribution.Id - -/** - * Execution plan for performing pure side-effects, i.e. returning no data to the user. - * - * @param name A name of the side-effect - * @param queryType The type of the query - * @param sideEffect The actual side-effect to be performed - */ -case class PureSideEffectExecutionPlan(name: String, queryType: InternalQueryType, sideEffect: QueryContext => Unit) - extends ExecutionPlan { - - override def run(ctx: QueryContext, planType: ExecutionMode, - params: MapValue): InternalExecutionResult = { - if (planType == ExplainMode) { - //close all statements - ctx.transactionalContext.close(success = true) - ExplainExecutionResult(Array.empty, description, queryType, Set.empty) - } else { - if (queryType == SCHEMA_WRITE) ctx.assertSchemaWritesAllowed() - - val countingCtx = new UpdateCountingQueryContext(ctx) - sideEffect(countingCtx) - ctx.transactionalContext.close(success = true) - PureSideEffectInternalExecutionResult(countingCtx, description, queryType, planType) - } - } - - private def description = PlanDescriptionImpl(Id.INVALID_ID, name, NoChildren, - Seq(Planner(ProcedurePlannerName.toTextOutput), - PlannerImpl(ProcedurePlannerName.name), - PlannerVersion(ProcedurePlannerName.version), - Runtime(ProcedureRuntimeName.toTextOutput), - RuntimeImpl(ProcedureRuntimeName.name), - Version(s"CYPHER ${CypherVersion.default.name}"), - RuntimeVersion(CypherVersion.default.name)) - , Set.empty) - - override def runtimeName: RuntimeName = ProcedureRuntimeName -} diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/PureSideEffectInternalExecutionResult.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/PureSideEffectInternalExecutionResult.scala deleted file mode 100644 index b031f8f7591b1..0000000000000 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/PureSideEffectInternalExecutionResult.scala +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2002-2018 "Neo4j," - * Neo4j Sweden AB [http://neo4j.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.procs - -import java.io.PrintWriter - -import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.StandardInternalExecutionResult -import org.neo4j.cypher.internal.compatibility.v3_5.runtime.ProcedureRuntimeName -import org.neo4j.cypher.internal.runtime._ -import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription -import org.neo4j.cypher.result.QueryResult.QueryResultVisitor -import org.neo4j.graphdb.Notification - -/** - * Empty result, as produced by a pure side-effect. - */ -case class PureSideEffectInternalExecutionResult(ctx: QueryContext, - executionPlanDescription: InternalPlanDescription, - queryType: InternalQueryType, - executionMode: ExecutionMode) - extends StandardInternalExecutionResult(ctx, ProcedureRuntimeName, None) - with StandardInternalExecutionResult.IterateByAccepting { - - override def fieldNames(): Array[String] = Array.empty - - override def accept[EX <: Exception](visitor: QueryResultVisitor[EX]): Unit = { - ctx.transactionalContext.close(success = true) - } - - override def queryStatistics(): QueryStatistics = ctx.getOptStatistics.getOrElse(QueryStatistics()) - - override def toList: List[Nothing] = List.empty - - override def dumpToString(writer: PrintWriter): Unit = { - writer.println("+-------------------+") - writer.println("| No data returned. |") - writer.println("+-------------------+") - writer.print(queryStatistics().toString) - } - - override def withNotifications(added: Notification*): InternalExecutionResult = this -} - - diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/SchemaWriteExecutionPlan.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/SchemaWriteExecutionPlan.scala new file mode 100644 index 0000000000000..f85e4dedf14b0 --- /dev/null +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/SchemaWriteExecutionPlan.scala @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2002-2018 "Neo4j," + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.procs + +import org.neo4j.cypher.internal.compatibility.v3_5.runtime._ +import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.ExecutionPlan +import org.neo4j.cypher.internal.runtime._ +import org.neo4j.cypher.internal.runtime.interpreted.UpdateCountingQueryContext +import org.neo4j.cypher.result.RuntimeResult +import org.neo4j.values.virtual.MapValue + +/** + * Execution plan for performing schema writes, i.e. creating or dropping indexes and constraints. + * + * @param name A name of the schema write + * @param schemaWrite The actual schema write to perform + */ +case class SchemaWriteExecutionPlan(name: String, schemaWrite: QueryContext => Unit) + extends ExecutionPlan { + + override def run(ctx: QueryContext, doProfile: Boolean, params: MapValue): RuntimeResult = { + + ctx.assertSchemaWritesAllowed() + + val countingCtx = new UpdateCountingQueryContext(ctx) + schemaWrite(countingCtx) + ctx.transactionalContext.close(success = true) + val runtimeResult = SchemaWriteRuntimeResult(countingCtx) + runtimeResult + } + + override def runtimeName: RuntimeName = ProcedureRuntimeName +} diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/SchemaWriteRuntimeResult.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/SchemaWriteRuntimeResult.scala new file mode 100644 index 0000000000000..64c6364524179 --- /dev/null +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/SchemaWriteRuntimeResult.scala @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2002-2018 "Neo4j," + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.procs + +import java.util + +import org.neo4j.cypher.internal.compatibility.v3_5.runtime.profiler.InterpretedProfileInformation +import org.neo4j.cypher.internal.runtime._ +import org.neo4j.cypher.result.QueryResult.QueryResultVisitor +import org.neo4j.cypher.result.{QueryProfile, RuntimeResult} +import org.neo4j.graphdb.ResourceIterator +import org.neo4j.helpers.collection.Iterators + +/** + * Empty result, as produced by a schema write. + */ +case class SchemaWriteRuntimeResult(ctx: QueryContext) extends RuntimeResult { + + override def fieldNames(): Array[String] = Array.empty + + override def accept[EX <: Exception](visitor: QueryResultVisitor[EX]): Unit = {} + + override def queryStatistics(): QueryStatistics = ctx.getOptStatistics.getOrElse(QueryStatistics()) + + override def isIterable: Boolean = true + + override def asIterator(): ResourceIterator[util.Map[String, AnyRef]] = Iterators.emptyResourceIterator() + + override def isExhausted: Boolean = true + + override def close(): Unit = {} + + override def queryProfile(): QueryProfile = QueryProfile.NONE +} + + diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/helpers/InternalWrapping.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/helpers/InternalWrapping.scala index e4c5f4a13136c..1156bc7f15f9f 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/helpers/InternalWrapping.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/helpers/InternalWrapping.scala @@ -29,6 +29,12 @@ import scala.collection.JavaConverters._ object InternalWrapping { def asKernelNotification(offset: Option[InputPosition])(notification: InternalNotification) = notification match { + case StartUnavailableFallback => + NotificationCode.START_UNAVAILABLE_FALLBACK.notification(graphdb.InputPosition.empty) + case CreateUniqueUnavailableFallback(pos) => + NotificationCode.CREATE_UNIQUE_UNAVAILABLE_FALLBACK.notification(pos.withOffset(offset).asInputPosition) + case RulePlannerUnavailableFallbackNotification => + NotificationCode.RULE_PLANNER_UNAVAILABLE_FALLBACK.notification(graphdb.InputPosition.empty) case DeprecatedStartNotification(pos, message) => NotificationCode.START_DEPRECATED.notification(pos.withOffset(offset).asInputPosition, NotificationDetail.Factory.message("START", message)) case CartesianProductNotification(pos, variables) => diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/helpers/RuntimeTextValueConverter.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/helpers/RuntimeTextValueConverter.scala index 67220e76a803f..a3ecb45dbb583 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/helpers/RuntimeTextValueConverter.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/helpers/RuntimeTextValueConverter.scala @@ -19,24 +19,27 @@ */ package org.neo4j.cypher.internal.compatibility.v3_5.runtime.helpers +import org.neo4j.cypher.internal.runtime.{QueryTransactionalContext, RuntimeScalaValueConverter} import org.neo4j.cypher.internal.runtime.interpreted.commands.values.KeyToken -import org.neo4j.cypher.internal.runtime.{QueryContext, RuntimeScalaValueConverter} +import org.neo4j.graphdb.Result.ResultRow import org.neo4j.graphdb.{Entity, Node, Path, Relationship} +import org.neo4j.internal.kernel.api.{PropertyCursor, Transaction} -import scala.collection.Map +import scala.collection.mutable.ArrayBuffer // Converts scala runtime values to human readable text // // Main use: Printing results when using ExecutionEngine // -class RuntimeTextValueConverter(scalaValues: RuntimeScalaValueConverter)(implicit context: QueryContext) { +class RuntimeTextValueConverter(scalaValues: RuntimeScalaValueConverter, + txContext: QueryTransactionalContext) { def asTextValue(a: Any): String = { val scalaValue = scalaValues.asShallowScalaValue(a) scalaValue match { case node: Node => s"Node[${node.getId}]${props(node)}" case relationship: Relationship => s":${relationship.getType.name()}[${relationship.getId}]${props(relationship)}" - case path: Path => path.toString + case path: Path => pathAsTextValue(path) case map: Map[_, _] => makeString(map) case opt: Option[_] => opt.map(asTextValue).getOrElse("None") case array: Array[_] => array.map(elem => asTextValue(elem)).mkString("[", ",", "]") @@ -50,18 +53,93 @@ class RuntimeTextValueConverter(scalaValues: RuntimeScalaValueConverter)(implici private def makeString(m: Map[_, _]) = m.map { case (k, v) => s"$k -> ${asTextValue(v)}" }.mkString("{", ", ", "}") + def dumpRowToString(columns: Array[String], row: ResultRow): Map[String, String] = { + val map = Map.newBuilder[String, String] + for (c <- columns) { + map += c -> asTextValue(row.get(c)) + } + map.result() + } + + def pathAsTextValue(path: Path): String = { + val nodes = path.nodes().iterator() + val relationships = path.relationships().iterator() + val tx = txContext.transaction + val sb = new StringBuilder + + def formatNode(n: Node) = { + val isDeleted = tx.dataRead().nodeDeletedInTransaction(n.getId) + val deletedString = if (isDeleted) ",deleted" else "" + sb ++= s"(${n.getId}$deletedString)" + } + + def formatRelationship(leftNode: Node, r: Relationship) = { + val isDeleted = tx.dataRead().relationshipDeletedInTransaction(r.getId) + val deletedString = if (isDeleted) ",deleted" else "" + if (r.getStartNode != leftNode) + sb += '<' + sb ++= s"-[${r.getId}:${r.getType}$deletedString]-" + if (r.getEndNode != leftNode) + sb += '>' + } + + var n = nodes.next() + formatNode(n) + while (relationships.hasNext) { + val r = relationships.next() + formatRelationship(n, r) + n = nodes.next() + formatNode(n) + } + + sb.result() + } + private def props(n: Node): String = { - val ops = context.nodeOps - val properties = if (isVirtualEntityHack(n)) Array.empty else ops.propertyKeyIds(n.getId) - val keyValStrings = properties.map(pkId => s"${context.getPropertyKeyName(pkId)}:${asTextValue(ops.getProperty(n.getId, pkId).asObject())}") - keyValStrings.mkString("{", ",", "}") + val tx = txContext.transaction + if (tx.dataRead().nodeDeletedInTransaction(n.getId)) { + return "{deleted}" + } + + if (!isVirtualEntityHack(n)) { + val nodeCursor = tx.cursors().allocateNodeCursor() + val propertyCursor = tx.cursors().allocatePropertyCursor() + tx.dataRead().singleNode(n.getId, nodeCursor) + if (nodeCursor.next()) { + nodeCursor.properties(propertyCursor) + return propertiesAsTextValue(propertyCursor) + } + } + "{}" } private def props(r: Relationship): String = { - val ops = context.relationshipOps - val properties = if (isVirtualEntityHack(r)) Array.empty else ops.propertyKeyIds(r.getId) - val keyValStrings = properties.map(pkId => s"${context.getPropertyKeyName(pkId)}:${asTextValue(ops.getProperty(r.getId, pkId).asObject())}") - keyValStrings.mkString("{", ",", "}") + val tx = txContext.transaction + if (tx.dataRead().relationshipDeletedInTransaction(r.getId)) { + return "{deleted}" + } + + if (!isVirtualEntityHack(r)) { + val relationshipCursor = tx.cursors().allocateRelationshipScanCursor() + val propertyCursor = tx.cursors().allocatePropertyCursor() + tx.dataRead().singleRelationship(r.getId, relationshipCursor) + if (relationshipCursor.next()) { + relationshipCursor.properties(propertyCursor) + return propertiesAsTextValue(propertyCursor) + } + } + "{}" + } + + private def propertiesAsTextValue(propertyCursor: PropertyCursor): String = { + val keyValues = new ArrayBuffer[String] + val tokenRead = txContext.transaction.tokenRead() + while (propertyCursor.next()) { + val key = tokenRead.propertyKeyName(propertyCursor.propertyKey()) + val value = asTextValue(propertyCursor.propertyValue().asObject()) + keyValues.append(s"$key:$value") + } + keyValues.mkString("{", ",", "}") } private def isVirtualEntityHack(entity:Entity): Boolean = entity.getId < 0 diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/profiler/InterpretedProfileInformation.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/profiler/InterpretedProfileInformation.scala new file mode 100644 index 0000000000000..789e831b9c16e --- /dev/null +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/profiler/InterpretedProfileInformation.scala @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2002-2018 "Neo4j," + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.cypher.internal.compatibility.v3_5.runtime.profiler + +import org.neo4j.cypher.result.{OperatorProfile, QueryProfile} +import org.opencypher.v9_0.util.attribution.Id + +import scala.collection.mutable + +class InterpretedProfileInformation extends QueryProfile { + + case class OperatorData(override val dbHits: Long, + override val rows: Long, + override val pageCacheHits: Long, + override val pageCacheMisses: Long) extends OperatorProfile { + + override def time: Long = -1L + } + + val pageCacheMap: mutable.Map[Id, PageCacheStats] = mutable.Map.empty + val dbHitsMap: mutable.Map[Id, ProfilingPipeQueryContext] = mutable.Map.empty + val rowMap: mutable.Map[Id, ProfilingIterator] = mutable.Map.empty + + def operatorProfile(operatorId: Int): OperatorProfile = { + val id = Id(operatorId) + val rows = rowMap.get(id).map(_.count).getOrElse(0L) + val dbHits = dbHitsMap.get(id).map(_.count).getOrElse(0L) + val pageCacheStats = pageCacheMap.getOrElse(id, PageCacheStats(0L, 0L)) + + OperatorData(dbHits, rows, pageCacheStats.hits, pageCacheStats.misses) + } +} + +case class PageCacheStats(hits: Long, misses: Long) + + diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/profiler/PlanDescriptionBuilder.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/profiler/PlanDescriptionBuilder.scala new file mode 100644 index 0000000000000..97f70b5350f57 --- /dev/null +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/profiler/PlanDescriptionBuilder.scala @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2002-2018 "Neo4j," + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.cypher.internal.compatibility.v3_5.runtime.profiler + +import org.neo4j.cypher.internal.compatibility.v3_5.runtime.RuntimeName +import org.neo4j.cypher.internal.planner.v3_5.spi.PlanningAttributes.Cardinalities +import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription.Arguments +import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription.Arguments.{Runtime, RuntimeImpl} +import org.neo4j.cypher.internal.runtime.planDescription.{InternalPlanDescription, LogicalPlan2PlanDescription} +import org.neo4j.cypher.internal.v3_5.logical.plans.LogicalPlan +import org.neo4j.cypher.result.QueryProfile +import org.opencypher.v9_0.frontend.PlannerName + +class PlanDescriptionBuilder(logicalPlan: LogicalPlan, + plannerName: PlannerName, + readOnly: Boolean, + cardinalities: Cardinalities, + runtimeName: RuntimeName) { + + def explain(): InternalPlanDescription = + LogicalPlan2PlanDescription(logicalPlan, plannerName, readOnly, cardinalities) + .addArgument(Runtime(runtimeName.toTextOutput)) + .addArgument(RuntimeImpl(runtimeName.name)) + + def profile(queryProfile: QueryProfile): InternalPlanDescription = { + + val planDescription = explain() + + planDescription map { + input: InternalPlanDescription => + val data = queryProfile.operatorProfile(input.id.x) + + input + .addArgument(Arguments.Rows(data.rows)) + .addArgument(Arguments.DbHits(data.dbHits)) + .addArgument(Arguments.PageCacheHits(data.pageCacheHits)) + .addArgument(Arguments.PageCacheMisses(data.pageCacheMisses)) + .addArgument(Arguments.PageCacheHitRatio(data.pageCacheHitRatio())) + .addArgument(Arguments.Time(data.time())) + } + } +} diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/profiler/Profiler.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/profiler/Profiler.scala index 0111641a8edb3..ac6caa7757965 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/profiler/Profiler.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/profiler/Profiler.scala @@ -23,10 +23,7 @@ import org.eclipse.collections.api.iterator.LongIterator import org.neo4j.cypher.internal.compatibility.v3_5.runtime.helpers.PrimitiveLongHelper import org.neo4j.cypher.internal.runtime.interpreted.pipes.{Pipe, PipeDecorator, QueryState} import org.neo4j.cypher.internal.runtime.interpreted.{DelegatingOperations, DelegatingQueryContext, ExecutionContext} -import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription -import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription.Arguments import org.neo4j.cypher.internal.runtime.{Operations, QueryContext} -import org.neo4j.helpers.MathUtil import org.neo4j.internal.kernel.api.helpers.RelationshipSelectionCursor import org.neo4j.kernel.impl.api.store.RelationshipIterator import org.neo4j.kernel.impl.factory.{DatabaseInfo, Edition} @@ -34,71 +31,52 @@ import org.neo4j.storageengine.api.RelationshipVisitor import org.neo4j.values.virtual.{NodeValue, RelationshipValue} import org.opencypher.v9_0.util.attribution.Id -import scala.collection.mutable - -class Profiler(databaseInfo: DatabaseInfo = DatabaseInfo.COMMUNITY) extends PipeDecorator { +class Profiler(databaseInfo: DatabaseInfo, + stats: InterpretedProfileInformation) extends PipeDecorator { outerProfiler => - val pageCacheStats: mutable.Map[Id, (Long, Long)] = mutable.Map.empty - val dbHitsStats: mutable.Map[Id, ProfilingPipeQueryContext] = mutable.Map.empty - val rowStats: mutable.Map[Id, ProfilingIterator] = mutable.Map.empty private var parentPipe: Option[Pipe] = None - def decorate(pipe: Pipe, iter: Iterator[ExecutionContext]): Iterator[ExecutionContext] = { - val oldCount = rowStats.get(pipe.id).map(_.count).getOrElse(0L) - val resultIter = new ProfilingIterator(iter, oldCount, pipe.id, if (trackPageCacheStats) updatePageCacheStatistics - else { - (_) => Unit}) - - rowStats(pipe.id) = resultIter + val oldCount = stats.rowMap.get(pipe.id).map(_.count).getOrElse(0L) + val resultIter = + new ProfilingIterator(iter, oldCount, pipe.id, + if (trackPageCacheStats) updatePageCacheStatistics + else _ => Unit + ) + + stats.rowMap(pipe.id) = resultIter resultIter } def decorate(pipe: Pipe, state: QueryState): QueryState = { - val decoratedContext = dbHitsStats.getOrElseUpdate(pipe.id, state.query match { + val decoratedContext = stats.dbHitsMap.getOrElseUpdate(pipe.id, state.query match { case p: ProfilingPipeQueryContext => new ProfilingPipeQueryContext(p.inner, pipe) case _ => new ProfilingPipeQueryContext(state.query, pipe) }) if (trackPageCacheStats) { val statisticProvider = decoratedContext.transactionalContext.kernelStatisticProvider - pageCacheStats(pipe.id) = (statisticProvider.getPageCacheHits, statisticProvider.getPageCacheMisses) + stats.pageCacheMap(pipe.id) = PageCacheStats(statisticProvider.getPageCacheHits, statisticProvider.getPageCacheMisses) } state.withQueryContext(decoratedContext) } - private def updatePageCacheStatistics(pipeId: Id) = { - val context = dbHitsStats(pipeId) + private def updatePageCacheStatistics(pipeId: Id): Unit = { + val context = stats.dbHitsMap(pipeId) val statisticProvider = context.transactionalContext.kernelStatisticProvider - val currentStat = pageCacheStats(pipeId) - pageCacheStats(pipeId) = (statisticProvider.getPageCacheHits - currentStat._1, statisticProvider.getPageCacheMisses - currentStat._2) + val currentStat = stats.pageCacheMap(pipeId) + stats.pageCacheMap(pipeId) = + PageCacheStats( + statisticProvider.getPageCacheHits - currentStat.hits, + statisticProvider.getPageCacheMisses - currentStat.misses + ) } private def trackPageCacheStats = { databaseInfo.edition != Edition.community } - def decorate(plan: () => InternalPlanDescription, verifyProfileReady: () => Unit): () => InternalPlanDescription = { - () => { - verifyProfileReady() - plan() map { - input: InternalPlanDescription => - val rows = rowStats.get(input.id).map(_.count).getOrElse(0L) - val dbHits = dbHitsStats.get(input.id).map(_.count).getOrElse(0L) - val (hits: Long, misses: Long) = pageCacheStats.getOrElse(input.id, (0L, 0L)) - val hitRatio = MathUtil.portion(hits, misses) - - input - .addArgument(Arguments.Rows(rows)) - .addArgument(Arguments.DbHits(dbHits)) - .addArgument(Arguments.PageCacheHits(hits)) - .addArgument(Arguments.PageCacheMisses(misses)) - .addArgument(Arguments.PageCacheHitRatio(hitRatio)) - } - } - } - def innerDecorator(owningPipe: Pipe): PipeDecorator = new PipeDecorator { innerProfiler => @@ -108,9 +86,6 @@ class Profiler(databaseInfo: DatabaseInfo = DatabaseInfo.COMMUNITY) extends Pipe outerProfiler.decorate(owningPipe, state) def decorate(pipe: Pipe, iter: Iterator[ExecutionContext]): Iterator[ExecutionContext] = iter - - def decorate(plan: () => InternalPlanDescription, verifyProfileReady: () => Unit): () => InternalPlanDescription = - outerProfiler.decorate(plan, verifyProfileReady) } def registerParentPipe(pipe: Pipe): Unit = @@ -204,11 +179,13 @@ class ProfilingIterator(inner: Iterator[ExecutionContext], startValue: Long, pip with Counter { _count = startValue + private var updatedStatistics = false def hasNext: Boolean = { val hasNext = inner.hasNext - if (!hasNext) { + if (!hasNext && !updatedStatistics) { updatePageCacheStatistics(pipeId) + updatedStatistics = true } hasNext } diff --git a/community/cypher/cypher/src/test/scala/org/neo4j/cypher/QueryPlanTestSupport.scala b/community/cypher/cypher/src/test/scala/org/neo4j/cypher/QueryPlanTestSupport.scala index 77bee6a25280f..99ce0f84a33af 100644 --- a/community/cypher/cypher/src/test/scala/org/neo4j/cypher/QueryPlanTestSupport.scala +++ b/community/cypher/cypher/src/test/scala/org/neo4j/cypher/QueryPlanTestSupport.scala @@ -19,8 +19,8 @@ */ package org.neo4j.cypher -import org.neo4j.cypher.internal.runtime.InternalExecutionResult import org.neo4j.cypher.planmatching.{CountInTree, ExactPlan, PlanInTree, PlanMatcher} +import org.neo4j.cypher.internal.RewindableExecutionResult import org.scalatest.matchers.{MatchResult, Matcher} trait QueryPlanTestSupport { @@ -66,10 +66,10 @@ trait QueryPlanTestSupport { def apply(name: String): PlanMatcher = haveAsRoot.aPlan(name) } - def haveCount(count: Int): Matcher[InternalExecutionResult] = new Matcher[InternalExecutionResult] { - override def apply(result: InternalExecutionResult): MatchResult = { + def haveCount(count: Int): Matcher[RewindableExecutionResult] = new Matcher[RewindableExecutionResult] { + override def apply(result: RewindableExecutionResult): MatchResult = { MatchResult( - matches = count == result.toList.length, + matches = count == result.size, rawFailureMessage = s"Result should have $count rows", rawNegatedFailureMessage = s"Result should not have $count rows") } diff --git a/community/cypher/cypher/src/test/scala/org/neo4j/cypher/QueryStatisticsTestSupport.scala b/community/cypher/cypher/src/test/scala/org/neo4j/cypher/QueryStatisticsTestSupport.scala index ff30d046cff7b..415a1f35b3d6f 100644 --- a/community/cypher/cypher/src/test/scala/org/neo4j/cypher/QueryStatisticsTestSupport.scala +++ b/community/cypher/cypher/src/test/scala/org/neo4j/cypher/QueryStatisticsTestSupport.scala @@ -19,99 +19,50 @@ */ package org.neo4j.cypher -import org.neo4j.cypher.internal.runtime.{InternalExecutionResult, QueryStatistics} -import org.neo4j.kernel.api.query.ExecutingQuery -import org.neo4j.kernel.impl.query.QueryExecutionMonitor +import org.neo4j.cypher.internal.RewindableExecutionResult +import org.neo4j.cypher.internal.runtime.QueryStatistics import org.scalatest.Assertions import org.scalatest.mock.MockitoSugar trait QueryStatisticsTestSupport extends MockitoSugar { self: Assertions => - implicit class QueryStatisticsAssertions(expected: QueryStatistics) { - def apply(actual: QueryStatistics) { - assertResult(expected)(actual) - } - - def apply(actual: InternalExecutionResult) { - implicit val monitor: QueryExecutionMonitor = new QueryExecutionMonitor { - override def endSuccess(query: ExecutingQuery){} - - override def endFailure(query: ExecutingQuery, throwable: Throwable){} - } - apply(actual.queryStatistics()) - } - } - - def assertStats( - result: InternalExecutionResult, - nodesCreated: Int = 0, - relationshipsCreated: Int = 0, - propertiesWritten: Int = 0, - nodesDeleted: Int = 0, - relationshipsDeleted: Int = 0, - labelsAdded: Int = 0, - labelsRemoved: Int = 0, - indexesAdded: Int = 0, - indexesRemoved: Int = 0, - uniqueConstraintsAdded: Int = 0, - uniqueConstraintsRemoved: Int = 0, - existenceConstraintsAdded: Int = 0, - existenceConstraintsRemoved: Int = 0, - nodekeyConstraintsAdded: Int = 0, - nodekeyConstraintsRemoved: Int = 0 + def assertStats(result: RewindableExecutionResult, + nodesCreated: Int = 0, + relationshipsCreated: Int = 0, + propertiesWritten: Int = 0, + nodesDeleted: Int = 0, + relationshipsDeleted: Int = 0, + labelsAdded: Int = 0, + labelsRemoved: Int = 0, + indexesAdded: Int = 0, + indexesRemoved: Int = 0, + uniqueConstraintsAdded: Int = 0, + uniqueConstraintsRemoved: Int = 0, + existenceConstraintsAdded: Int = 0, + existenceConstraintsRemoved: Int = 0, + nodekeyConstraintsAdded: Int = 0, + nodekeyConstraintsRemoved: Int = 0 ): Unit = { - assertStatsResult( - nodesCreated, - relationshipsCreated, - propertiesWritten, - nodesDeleted, - relationshipsDeleted, - labelsAdded, - labelsRemoved, - indexesAdded, - indexesRemoved, - uniqueConstraintsAdded, - uniqueConstraintsRemoved, - existenceConstraintsAdded, - existenceConstraintsRemoved, - nodekeyConstraintsAdded, - nodekeyConstraintsRemoved - )(result) - } + val expected = + QueryStatistics( + nodesCreated, + relationshipsCreated, + propertiesWritten, + nodesDeleted, + relationshipsDeleted, + labelsAdded, + labelsRemoved, + indexesAdded, + indexesRemoved, + uniqueConstraintsAdded, + uniqueConstraintsRemoved, + existenceConstraintsAdded, + existenceConstraintsRemoved, + nodekeyConstraintsAdded, + nodekeyConstraintsRemoved + ) - // This api is more in line with scala test assertions which prefer the expectation before the actual - def assertStatsResult(nodesCreated: Int = 0, - relationshipsCreated: Int = 0, - propertiesWritten: Int = 0, - nodesDeleted: Int = 0, - relationshipsDeleted: Int = 0, - labelsAdded: Int = 0, - labelsRemoved: Int = 0, - indexesAdded: Int = 0, - indexesRemoved: Int = 0, - uniqueConstraintsAdded: Int = 0, - uniqueConstraintsRemoved: Int = 0, - existenceConstraintsAdded: Int = 0, - existenceConstraintsRemoved: Int = 0, - nodekeyConstraintsAdded: Int = 0, - nodekeyConstraintsRemoved: Int = 0 - ): QueryStatisticsAssertions = - QueryStatistics( - nodesCreated, - relationshipsCreated, - propertiesWritten, - nodesDeleted, - relationshipsDeleted, - labelsAdded, - labelsRemoved, - indexesAdded, - indexesRemoved, - uniqueConstraintsAdded, - uniqueConstraintsRemoved, - existenceConstraintsAdded, - existenceConstraintsRemoved, - nodekeyConstraintsAdded, - nodekeyConstraintsRemoved - ) + assertResult(expected)(result.queryStatistics()) + } } diff --git a/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/RewindableExecutionResult.scala b/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/RewindableExecutionResult.scala index 43ab4ad5c83cd..827f927df5f23 100644 --- a/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/RewindableExecutionResult.scala +++ b/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/RewindableExecutionResult.scala @@ -19,98 +19,91 @@ */ package org.neo4j.cypher.internal -import org.neo4j.cypher.exceptionHandler -import org.neo4j.cypher.internal.compatibility.ClosingExecutionResult -import org.neo4j.cypher.internal.compatibility.v2_3.{ExecutionResultWrapper => ExecutionResultWrapperFor2_3, exceptionHandler => exceptionHandlerFor2_3} -import org.neo4j.cypher.internal.compatibility.v3_1.{ExecutionResultWrapper => ExecutionResultWrapperFor3_1, exceptionHandler => exceptionHandlerFor3_1} -import org.neo4j.cypher.internal.compatibility.v3_5.runtime._ -import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan._ -import org.neo4j.cypher.internal.compiler.{v2_3, v3_1} import org.neo4j.cypher.internal.javacompat.ExecutionResult import org.neo4j.cypher.internal.runtime._ -import org.neo4j.graphdb.{QueryExecutionType, Result} +import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription +import org.neo4j.cypher.result.QueryResult.QueryResultVisitor +import org.neo4j.cypher.result.{QueryResult, RuntimeResult} +import org.neo4j.graphdb.Result.{ResultRow, ResultVisitor} +import org.neo4j.graphdb.{Notification, Result} -object RewindableExecutionResult { +import scala.collection.mutable.ArrayBuffer - private def eagerize(inner: InternalExecutionResult): InternalExecutionResult = - inner match { - case other: PipeExecutionResult => - exceptionHandler.runSafely { - new PipeExecutionResult(other.result.toEager, other.columns.toArray, other.state, other.executionPlanBuilder, - other.executionMode, READ_WRITE) - } - case other: StandardInternalExecutionResult => - exceptionHandler.runSafely { - other.toEagerResultForTestingOnly() - } +/** + * Helper class to ease asserting on cypher results from scala. + */ +class RewindableExecutionResult(val columns: Array[String], + result: Seq[Map[String, AnyRef]], + val executionMode: ExecutionMode, + planDescription: InternalPlanDescription, + statistics: QueryStatistics, + val notifications: Iterable[Notification]) { - case _ => - inner - } + def columnAs[T](column: String): Iterator[T] = result.iterator.map(row => row(column).asInstanceOf[T]) + def toList: Seq[Map[String, AnyRef]] = result + def toSet: Set[Map[String, AnyRef]] = result.toSet + def size: Long = result.size + def head(str: String): AnyRef = result.head(str) - private def eagerize(inner: v2_3.executionplan.InternalExecutionResult): v2_3.executionplan.InternalExecutionResult = { - inner match { - case other: v2_3.PipeExecutionResult => - exceptionHandlerFor2_3.runSafely { - new v2_3.PipeExecutionResult(other.result.toEager, other.columns, other.state, other.executionPlanBuilder, - other.executionMode, QueryExecutionType.QueryType.READ_WRITE) - } - case other: v2_3.ExplainExecutionResult => - v2_3.ExplainExecutionResult(other.columns, other.executionPlanDescription, other.executionType.queryType(), other.notifications) - case _ => - inner - } - } + def executionPlanDescription(): InternalPlanDescription = planDescription - private def eagerize(inner: v3_1.executionplan.InternalExecutionResult): v3_1.executionplan.InternalExecutionResult = { - inner match { - case other: v3_1.PipeExecutionResult => - exceptionHandlerFor3_1.runSafely { - new v3_1.PipeExecutionResult(other.result.toEager, other.columns, other.state, other.executionPlanBuilder, - other.executionMode, v3_1.executionplan.READ_WRITE) - } - case other: v3_1.ExplainExecutionResult => - v3_1.ExplainExecutionResult(other.columns, other.executionPlanDescription, other.executionType, other.notifications) - case _ => - inner - } - } + def executionPlanString(): String = planDescription.toString - private class CachingExecutionResultWrapperFor3_1(inner: v3_1.executionplan.InternalExecutionResult, - planner: v3_1.PlannerName, - runtime: v3_1.RuntimeName, - preParsingNotification: Set[org.neo4j.graphdb.Notification], - offset: Option[frontend.v3_1.InputPosition]) - extends ExecutionResultWrapperFor3_1(inner, planner, runtime, preParsingNotification, offset) { - val cache: List[Map[String, Any]] = inner.toList - override val toList: List[Map[String, Any]] = cache - } + def queryStatistics(): QueryStatistics = statistics - private class CachingExecutionResultWrapperFor2_3(inner: v2_3.executionplan.InternalExecutionResult, - planner: v2_3.PlannerName, - runtime: v2_3.RuntimeName, - preParsingNotification: Set[org.neo4j.graphdb.Notification], - offset: Option[frontend.v2_3.InputPosition]) - extends ExecutionResultWrapperFor2_3(inner, planner, runtime, preParsingNotification, offset) { - val cache: List[Map[String, Any]] = inner.toList - override val toList: List[Map[String, Any]] = cache - } + def isEmpty: Boolean = result.isEmpty +} - def apply(in: Result): InternalExecutionResult = { +object RewindableExecutionResult { + + val scalaValues = new RuntimeScalaValueConverter(isGraphKernelResultValue) + + def apply(in: Result): RewindableExecutionResult = { val internal = in.asInstanceOf[ExecutionResult].internalExecutionResult - .asInstanceOf[ClosingExecutionResult].inner apply(internal) } - def apply(internal: InternalExecutionResult) : InternalExecutionResult = { - internal match { - case ExecutionResultWrapperFor3_1(inner, planner, runtime, preParsingNotification, offset) => - val wrapper = new CachingExecutionResultWrapperFor3_1(eagerize(inner), planner, runtime, preParsingNotification, offset) - exceptionHandlerFor3_1.runSafely(wrapper) - case ExecutionResultWrapperFor2_3(inner, planner, runtime, preParsingNotification, offset) => - val wrapper = new CachingExecutionResultWrapperFor2_3(eagerize(inner), planner, runtime, preParsingNotification, offset) - exceptionHandlerFor2_3.runSafely(wrapper) - case _ => exceptionHandler.runSafely(eagerize(internal)) - } + def apply(runtimeResult: RuntimeResult, queryContext: QueryContext): RewindableExecutionResult = { + val result = new ArrayBuffer[Map[String, AnyRef]]() + val columns = runtimeResult.fieldNames() + + runtimeResult.accept(new QueryResultVisitor[Exception] { + override def visit(row: QueryResult.Record): Boolean = { + val map = new java.util.HashMap[String, AnyRef]() + val values = row.fields() + for (i <- columns.indices) { + map.put(columns(i), queryContext.asObject(values(i))) + } + result += scalaValues.asDeepScalaMap(map).asInstanceOf[Map[String, AnyRef]] + true + } + }) + + new RewindableExecutionResult(columns, result, NormalMode, null, runtimeResult.queryStatistics(), Seq.empty) + } + + def apply(internal: InternalExecutionResult) : RewindableExecutionResult = { + val result = new ArrayBuffer[Map[String, AnyRef]]() + val columns = internal.fieldNames() + + internal.accept(new ResultVisitor[Exception] { + override def visit(row: ResultRow): Boolean = { + val map = new java.util.HashMap[String, AnyRef]() + for (c <- columns) { + map.put(c, row.get(c)) + } + result += scalaValues.asDeepScalaMap(map).asInstanceOf[Map[String, AnyRef]] + true + } + }) + + new RewindableExecutionResult( + columns, + result.toList, + internal.executionMode, + internal.executionPlanDescription(), + internal.queryStatistics(), + internal.notifications + ) } } diff --git a/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/ClosingIteratorTest.scala b/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/ClosingIteratorTest.scala deleted file mode 100644 index 37852088a2769..0000000000000 --- a/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/ClosingIteratorTest.scala +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright (c) 2002-2018 "Neo4j," - * Neo4j Sweden AB [http://neo4j.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.cypher.internal.compatibility.v3_5.runtime - -import org.mockito.ArgumentMatchers.anyBoolean -import org.mockito.Mockito._ -import org.opencypher.v9_0.util.{CypherException, TaskCloser} -import org.opencypher.v9_0.util.test_helpers.CypherFunSuite -import org.neo4j.values.AnyValue -import org.neo4j.values.storable.Values.intValue - -class ClosingIteratorTest extends CypherFunSuite { - var taskCloser: TaskCloser = _ - val exceptionDecorator: CypherException => CypherException = identity - - override def beforeEach() { - super.beforeEach() - taskCloser = mock[TaskCloser] - } - - test("should not close prematurely") { - //Given - val wrappee = Iterator(Map("k" -> intValue(42))) - val iterator = new ClosingIterator(wrappee, taskCloser, exceptionDecorator) - //When - val result = iterator.next() - - //Then - verify(taskCloser, never()).close(anyBoolean()) - result should equal(Map[String, Any]("k" -> intValue(42))) - } - - test("should cleanup even for empty iterator") { - //Given - val wrappee = Iterator.empty - val iterator = new ClosingIterator(wrappee, taskCloser, exceptionDecorator) - - //When - val result = iterator.hasNext - - //Then - verify(taskCloser).close(success = true) - result should equal(false) - } - - test("multiple has next should not close more than once") { - //Given - val wrappee = Iterator.empty - val iterator = new ClosingIterator(wrappee, taskCloser, exceptionDecorator) - - //When - val result = iterator.hasNext - iterator.hasNext - iterator.hasNext - iterator.hasNext - iterator.hasNext - - //Then - verify(taskCloser, atLeastOnce()).close(success = true) - result shouldBe false - } - - test("exception in hasNext should fail transaction") { - //Given - val wrappee = mock[Iterator[Map[String, AnyValue]]] - when(wrappee.hasNext).thenThrow(new RuntimeException) - val iterator = new ClosingIterator(wrappee, taskCloser, exceptionDecorator) - - //When - intercept[RuntimeException](iterator.hasNext) - - //Then - verify(taskCloser).close(success = false) - } - - test("exception in next should fail transaction") { - //Given - val wrappee = mock[Iterator[Map[String, AnyValue]]] - when(wrappee.hasNext).thenReturn(true) - when(wrappee.next()).thenThrow(new RuntimeException) - - val iterator = new ClosingIterator(wrappee, taskCloser, exceptionDecorator) - - //When - intercept[RuntimeException](iterator.next()) - - //Then - verify(taskCloser).close(success = false) - } - - test("close runs cleanup") { - //Given - val wrappee = Iterator(Map("k" -> intValue(42)), Map("k" -> intValue(43))) - val iterator = new ClosingIterator(wrappee, taskCloser, exceptionDecorator) - - //When - val result = iterator.next() - iterator.close() - - //Then - verify(taskCloser).close(success = true) - result should equal(Map[String, AnyValue]("k" -> intValue(42))) - } -} diff --git a/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/PipeExecutionPlanBuilderIT.scala b/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/PipeExecutionPlanBuilderIT.scala index ea6d8a65030c6..e4864d3939975 100644 --- a/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/PipeExecutionPlanBuilderIT.scala +++ b/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/PipeExecutionPlanBuilderIT.scala @@ -205,6 +205,6 @@ class PipeExecutionPlanBuilderIT extends CypherFunSuite with LogicalPlanningTest mutable.Map("existing1" -> RelTypeId(1), "existing2" -> RelTypeId(2), "existing3" -> RelTypeId(3))) - PipeExecutionBuilderContext(semanticTable, readOnly = true, mock[Cardinalities]) + PipeExecutionBuilderContext(semanticTable, readOnly = true) } } diff --git a/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/ExecutionWorkflowBuilderTest.scala b/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/ExecutionWorkflowBuilderTest.scala deleted file mode 100644 index e8d79887285a1..0000000000000 --- a/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/ExecutionWorkflowBuilderTest.scala +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (c) 2002-2018 "Neo4j," - * Neo4j Sweden AB [http://neo4j.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan - -import org.mockito.ArgumentMatchers._ -import org.mockito.Mockito._ -import org.neo4j.cypher.internal.compatibility.v3_5.runtime.{EagerResultIterator, _} -import org.neo4j.cypher.internal.compiler.v3_5.planner.LogicalPlanConstructionTestSupport -import org.opencypher.v9_0.frontend.phases.{InternalNotificationLogger, devNullLogger} -import org.neo4j.cypher.internal.planner.v3_5.spi.IDPPlannerName -import org.neo4j.cypher.internal.runtime.interpreted.pipes.Pipe -import org.neo4j.cypher.internal.runtime._ -import org.opencypher.v9_0.util.test_helpers.CypherFunSuite -import org.neo4j.cypher.internal.v3_5.logical.plans.Argument -import org.neo4j.values.virtual.MapValue -import org.neo4j.values.virtual.VirtualValues.EMPTY_MAP - -class ExecutionWorkflowBuilderTest extends CypherFunSuite with LogicalPlanConstructionTestSupport { - - private val PlannerName = IDPPlannerName - private val logicalPlan = Argument() - - test("produces eager results for updating queries") { - // GIVEN - val pipe = mock[Pipe] - when(pipe.createResults(any())).thenReturn(Iterator.empty) - val context = mock[QueryContext] - when(context.transactionalContext).thenReturn(mock[QueryTransactionalContext]) - when(context.resources).thenReturn(mock[CloseableResource]) - - val builderFactory = InterpretedExecutionResultBuilderFactory(pipe, readOnly = false, List.empty, logicalPlan) - - // WHEN - val builder = builderFactory.create() - builder.setQueryContext(context) - - // THEN - val result = build(builder, NormalMode, EMPTY_MAP, devNullLogger, InterpretedRuntimeName, readOnly = false) - result shouldBe a [PipeExecutionResult] - result.asInstanceOf[PipeExecutionResult].result shouldBe a[EagerResultIterator] - } - - test("produces lazy results for non-updating queries") { - val pipe = mock[Pipe] - when(pipe.createResults(any())).thenReturn(Iterator.empty) - val context = mock[QueryContext] - val builderFactory = new InterpretedExecutionResultBuilderFactory(pipe, readOnly = true, List.empty, logicalPlan) - - // WHEN - val builder = builderFactory.create() - builder.setQueryContext(context) - - // THEN - val result = build(builder, NormalMode, EMPTY_MAP, devNullLogger, InterpretedRuntimeName, readOnly = true) - result shouldBe a [PipeExecutionResult] - result.asInstanceOf[PipeExecutionResult].result should not be an[EagerResultIterator] - } - - test("produces explain results for EXPLAIN queries") { - // GIVEN - val pipe = mock[Pipe] - when(pipe.createResults(any())).thenReturn(Iterator.empty) - val context = mock[QueryContext] - when(context.transactionalContext).thenReturn(mock[QueryTransactionalContext]) - when(context.resources).thenReturn(mock[CloseableResource]) - val builderFactory = InterpretedExecutionResultBuilderFactory(pipe, readOnly = true, List.empty, logicalPlan) - - // WHEN - val builder = builderFactory.create() - builder.setQueryContext(context) - - // THEN - val result = build(builder, ExplainMode, EMPTY_MAP, devNullLogger, InterpretedRuntimeName, readOnly = true) - result shouldBe a [ExplainExecutionResult] - } - - private def build(builder: ExecutionResultBuilder, - planType: ExecutionMode, - params: MapValue, - notificationLogger: InternalNotificationLogger, - runtimeName: RuntimeName, - readOnly: Boolean) = - builder.build(planType, params, notificationLogger, PlannerName, runtimeName, readOnly, new StubCardinalities) -} diff --git a/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/StandardInternalExecutionResultTest.scala b/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/StandardInternalExecutionResultTest.scala new file mode 100644 index 0000000000000..af0dfb3936586 --- /dev/null +++ b/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/StandardInternalExecutionResultTest.scala @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2002-2018 "Neo4j," + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan + +import java.util + +import org.neo4j.cypher.result.RuntimeResult +import org.neo4j.function.ThrowingBiConsumer +import org.neo4j.graphdb.NotFoundException +import org.neo4j.graphdb.Result.{ResultRow, ResultVisitor} +import org.neo4j.helpers.collection.Iterators +import org.neo4j.values.AnyValue +import org.neo4j.values.storable._ +import org.neo4j.values.virtual.{ListValue, MapValue} +import org.opencypher.v9_0.util.test_helpers.CypherFunSuite + +import scala.collection.JavaConverters + +// We're going to get back here. +class StandardInternalExecutionResultTestTODO extends CypherFunSuite { + + def runtimeResult(row: util.Map[String, Any] = new util.HashMap()): RuntimeResult = ??? + + def executionResult(row: util.Map[String, Any] = new util.HashMap()): StandardInternalExecutionResult = ??? + + test("should return scala objects") { + val result = executionResult(javaMap("foo" -> "", "bar" -> "")) + + result.fieldNames() should equal(List("foo", "bar")) + } + + test("should return java objects for string") { + val result = executionResult(javaMap("foo" -> "bar")) + + Iterators.asList(result.javaColumnAs[String]("foo")) should equal(javaList("bar")) + } + + test("should return java objects for list") { + val result = executionResult(javaMap("foo" -> javaList(42L))) + + Iterators.asList(result.javaColumnAs[List[Integer]]("foo")) should equal(javaList(javaList(42L))) + } + + test("should return java objects for map") { + val result = executionResult(javaMap("foo" -> javaMap("key" -> "value"))) + + Iterators.asList(result.javaColumnAs[Map[String, Any]]("foo")) should equal(javaList(javaMap("key" -> "value"))) + } + + test("should throw if non-existing column") { + val result = executionResult(javaMap("foo" -> "bar")) + + a [NotFoundException] shouldBe thrownBy(result.javaColumnAs[String]("baz")) + } + + test("should return a java iterator for string") { + val result = executionResult(javaMap("foo" -> "bar")) + + Iterators.asList(result.javaIterator) should equal(javaList(javaMap("foo" -> "bar"))) + } + + test("should return a java iterator for list") { + val result = executionResult(javaMap("foo" -> javaList(42L))) + + Iterators.asList(result.javaIterator) should equal(javaList(javaMap("foo" -> javaList(42L)))) + } + + test("should return a java iterator for map") { + val result = executionResult(javaMap("foo" -> javaMap("key" -> "value"))) + + Iterators.asList(result.javaIterator) should equal(javaList(javaMap("foo" -> javaMap("key" -> "value")))) + } + + test("javaIterator hasNext should not call accept if results already consumed") { + // given + val result = executionResult() + + // when + result.accept(new ResultVisitor[Exception] { + override def visit(row: ResultRow): Boolean = { + false + } + }) + + // then + result.javaIterator.hasNext should be(false) + } + + test("close should work after result is consumed") { + // given + val result = executionResult(javaMap("a" -> "1", "b" -> "2")) + + // when + result.accept(new ResultVisitor[Exception] { + override def visit(row: ResultRow): Boolean = { + true + } + }) + + result.close() + + // then + // call of close actually worked + } + + import JavaConverters._ + private def toObjectConverter(a: AnyRef): AnyRef = a match { + case Values.NO_VALUE => null + case s: TextValue => s.stringValue() + case b: BooleanValue => Boolean.box(b.booleanValue()) + case f: FloatingPointValue => Double.box(f.doubleValue()) + case i: IntegralValue => Long.box(i.longValue()) + case l: ListValue => + val list = new util.ArrayList[AnyRef] + l.iterator().asScala.foreach(a => list.add(toObjectConverter(a))) + list + case m: MapValue => + val map = new util.HashMap[String, AnyRef]() + m.foreach(new ThrowingBiConsumer[String, AnyValue, RuntimeException] { + override def accept(t: String, u: AnyValue): Unit = map.put(t, toObjectConverter(u)) + }) + map + } + + private def javaList[T](elements: T*): util.List[T] = elements.toList.asJava + + private def javaMap[K, V](pairs: (K, V)*): util.Map[K, V] = pairs.toMap.asJava + +} diff --git a/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/ProcedureCallExecutionPlanTest.scala b/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/ProcedureCallExecutionPlanTest.scala index f5aeeb417bb25..f057d9938e5a0 100644 --- a/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/ProcedureCallExecutionPlanTest.scala +++ b/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/executionplan/procs/ProcedureCallExecutionPlanTest.scala @@ -24,15 +24,18 @@ import org.mockito.Mockito.when import org.mockito.invocation.InvocationOnMock import org.mockito.stubbing.Answer import org.neo4j.cypher.internal.runtime.interpreted.commands.convert.{CommunityExpressionConverter, ExpressionConverters} -import org.neo4j.cypher.internal.runtime.{CloseableResource, NormalMode, QueryContext, QueryTransactionalContext} -import org.opencypher.v9_0.util.DummyPosition -import org.opencypher.v9_0.util.symbols._ -import org.opencypher.v9_0.util.test_helpers.CypherFunSuite -import org.opencypher.v9_0.expressions._ +import org.neo4j.cypher.internal.runtime.{CloseableResource, QueryContext, QueryTransactionalContext} import org.neo4j.cypher.internal.v3_5.logical.plans._ +import org.neo4j.cypher.result.RuntimeResult import org.neo4j.internal.kernel.api.Procedures import org.neo4j.values.storable.LongValue import org.neo4j.values.virtual.VirtualValues.EMPTY_MAP +import org.opencypher.v9_0.expressions._ +import org.opencypher.v9_0.util.DummyPosition +import org.opencypher.v9_0.util.symbols._ +import org.opencypher.v9_0.util.test_helpers.CypherFunSuite + +import scala.collection.JavaConverters._ class ProcedureCallExecutionPlanTest extends CypherFunSuite { @@ -41,23 +44,23 @@ class ProcedureCallExecutionPlanTest extends CypherFunSuite { test("should be able to call procedure with single argument") { // Given val proc = ProcedureCallExecutionPlan(readSignature, Seq(add(int(42), int(42))), Seq("b" -> CTInteger), Seq(0 -> "b"), - notifications = Set.empty, converters) + converters) // When - val res = proc.run(ctx, NormalMode, EMPTY_MAP) + val res = proc.run(ctx, false, EMPTY_MAP) // Then - res.toList should equal(List(Map("b" -> 84))) + toList(res) should equal(List(Map("b" -> 84))) } test("should eagerize write procedure") { // Given val proc = ProcedureCallExecutionPlan(writeSignature, Seq(add(int(42), int(42))), Seq("b" -> CTInteger), Seq(0 -> "b"), - notifications = Set.empty, converters) + converters) // When - proc.run(ctx, NormalMode, EMPTY_MAP) + proc.run(ctx, false, EMPTY_MAP) // Then without touching the result, it should have been spooled out iteratorExhausted should equal(true) @@ -67,10 +70,10 @@ class ProcedureCallExecutionPlanTest extends CypherFunSuite { // Given val proc = ProcedureCallExecutionPlan(readSignature, Seq(add(int(42), int(42))), Seq("b" -> CTInteger), Seq(0 -> "b"), - notifications = Set.empty, converters) + converters) // When - proc.run(ctx, NormalMode, EMPTY_MAP) + proc.run(ctx, false, EMPTY_MAP) // Then without touching the result, the Kernel iterator should not be touched iteratorExhausted should equal(false) @@ -100,6 +103,9 @@ class ProcedureCallExecutionPlanTest extends CypherFunSuite { ProcedureReadWriteAccess(Array.empty) ) + private def toList(res: RuntimeResult): List[Map[String, AnyRef]] = + res.asIterator().asScala.map(_.asScala.toMap).toList + private val pos = DummyPosition(-1) val ctx = mock[QueryContext] when(ctx.resources).thenReturn(mock[CloseableResource]) diff --git a/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/profiler/ProfilerTest.scala b/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/profiler/ProfilerTest.scala index 5850620fdbc0c..53024411c3d4e 100644 --- a/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/profiler/ProfilerTest.scala +++ b/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/compatibility/v3_5/runtime/profiler/ProfilerTest.scala @@ -25,9 +25,8 @@ import org.neo4j.cypher.internal.planner.v3_5.spi.{EmptyKernelStatisticProvider, import org.neo4j.cypher.internal.runtime.interpreted.commands.expressions.{NestedPipeExpression, ProjectedPath} import org.neo4j.cypher.internal.runtime.interpreted.pipes._ import org.neo4j.cypher.internal.runtime.interpreted.{ExecutionContext, QueryStateHelper} -import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription.Arguments._ -import org.neo4j.cypher.internal.runtime.planDescription.{PlanDescriptionImpl, _} import org.neo4j.cypher.internal.runtime.{QueryContext, QueryTransactionalContext} +import org.neo4j.cypher.result.{OperatorProfile, QueryProfile} import org.neo4j.kernel.impl.factory.DatabaseInfo import org.neo4j.values.storable.Values.NO_VALUE import org.opencypher.v9_0.util.attribution.{Id, SequentialIdGen} @@ -42,16 +41,15 @@ class ProfilerTest extends CypherFunSuite { val start = ArgumentPipe()(idGen.id()) val pipe = ProfilerTestPipe(start, "foo", rows = 10, dbAccess = 20)(idGen.id()) val queryContext: QueryContext = prepareQueryContext() - val profiler = new Profiler(DatabaseInfo.ENTERPRISE) + val profile = new InterpretedProfileInformation + val profiler = new Profiler(DatabaseInfo.ENTERPRISE, profile) val queryState = QueryStateHelper.emptyWith(query = queryContext, decorator = profiler) - val planDescription = createPlanDescription("single row" -> start, "foo" -> pipe) //WHEN materialize(pipe.createResults(queryState)) - val decoratedResult = profiler.decorate(() => planDescription, verifyProfileReady = () => {}) //THEN - assertRecorded(decoratedResult(), "foo", expectedRows = 10, expectedDbHits = 20) + assertRecorded(profile, pipe.id, expectedRows = 10, expectedDbHits = 20) } test("report page cache statistics for simplest case") { @@ -60,26 +58,18 @@ class ProfilerTest extends CypherFunSuite { val statisticProvider = new ConfiguredKernelStatisticProvider() val pipe = ProfilerTestPipe(start, "foo", rows = 10, dbAccess = 20, statisticProvider, hits = 2, misses = 7)(idGen.id()) val queryContext: QueryContext = prepareQueryContext(statisticProvider) - val profiler = new Profiler(DatabaseInfo.ENTERPRISE) + val profile = new InterpretedProfileInformation + val profiler = new Profiler(DatabaseInfo.ENTERPRISE, profile) val queryState = QueryStateHelper.emptyWith(query = queryContext, decorator = profiler) - val planDescription = createPlanDescription("single row" -> start, "foo" -> pipe) //WHEN materialize(pipe.createResults(queryState)) - val decoratedResult = profiler.decorate(() => planDescription, verifyProfileReady = () => {}) //THEN - assertRecorded(decoratedResult(), "foo", expectedRows = 10, expectedDbHits = 20, + assertRecorded(profile, pipe.id, expectedRows = 10, expectedDbHits = 20, expectedPageCacheHits = 2, expectedPageCacheMisses = 7) } - private def createPlanDescription(first: (String, Pipe), tail: (String, Pipe)*): InternalPlanDescription = { - val firstDescr: InternalPlanDescription = PlanDescriptionImpl(first._2.id, first._1, NoChildren, Seq.empty, Set.empty) - tail.foldLeft(firstDescr) { - case (descr, (name, pipe)) => PlanDescriptionImpl(pipe.id, name, SingleChild(descr), Seq.empty, Set.empty) - } - } - test("should report multiple pipes case") { //GIVEN val start = ArgumentPipe()(idGen.id()) @@ -87,18 +77,17 @@ class ProfilerTest extends CypherFunSuite { val pipe2 = ProfilerTestPipe(pipe1, "bar", rows = 20, dbAccess = 40)(idGen.id()) val pipe3 = ProfilerTestPipe(pipe2, "baz", rows = 1, dbAccess = 2)(idGen.id()) val queryContext: QueryContext = prepareQueryContext() - val profiler = new Profiler + val profile = new InterpretedProfileInformation + val profiler = new Profiler(DatabaseInfo.COMMUNITY, profile) val queryState = QueryStateHelper.emptyWith(query = queryContext, decorator = profiler) - val planDescription = createPlanDescription("single row" -> start, "foo" -> pipe1, "bar" -> pipe2, "baz" -> pipe3) //WHEN materialize(pipe3.createResults(queryState)) - val decoratedResult = profiler.decorate(() => planDescription, verifyProfileReady = () => {}) //THEN - assertRecorded(decoratedResult(), "foo", expectedRows = 10, expectedDbHits = 25) - assertRecorded(decoratedResult(), "bar", expectedRows = 20, expectedDbHits = 40) - assertRecorded(decoratedResult(), "baz", expectedRows = 1, expectedDbHits = 2) + assertRecorded(profile, pipe1.id, expectedRows = 10, expectedDbHits = 25) + assertRecorded(profile, pipe2.id, expectedRows = 20, expectedDbHits = 40) + assertRecorded(profile, pipe3.id, expectedRows = 1, expectedDbHits = 2) } test("report page cache statistic for multiple pipes case") { @@ -109,18 +98,17 @@ class ProfilerTest extends CypherFunSuite { val pipe2 = ProfilerTestPipe(pipe1, "bar", rows = 20, dbAccess = 40, statisticProvider, 12, 35)(idGen.id()) val pipe3 = ProfilerTestPipe(pipe2, "baz", rows = 1, dbAccess = 2, statisticProvider, 37, 68)(idGen.id()) val queryContext: QueryContext = prepareQueryContext(statisticProvider) - val profiler = new Profiler(DatabaseInfo.ENTERPRISE) + val profile = new InterpretedProfileInformation + val profiler = new Profiler(DatabaseInfo.ENTERPRISE, profile) val queryState = QueryStateHelper.emptyWith(query = queryContext, decorator = profiler) - val planDescription = createPlanDescription("single row" -> start, "foo" -> pipe1, "bar" -> pipe2, "baz" -> pipe3) //WHEN materialize(pipe3.createResults(queryState)) - val decoratedResult = profiler.decorate(() => planDescription, verifyProfileReady = () => {}) //THEN - assertRecorded(decoratedResult(), "foo", expectedRows = 10, expectedDbHits = 25, expectedPageCacheHits = 2, expectedPageCacheMisses = 7) - assertRecorded(decoratedResult(), "bar", expectedRows = 20, expectedDbHits = 40, expectedPageCacheHits = 10, expectedPageCacheMisses = 28) - assertRecorded(decoratedResult(), "baz", expectedRows = 1, expectedDbHits = 2, expectedPageCacheHits = 25, expectedPageCacheMisses = 33) + assertRecorded(profile, pipe1.id, expectedRows = 10, expectedDbHits = 25, expectedPageCacheHits = 2, expectedPageCacheMisses = 7) + assertRecorded(profile, pipe2.id, expectedRows = 20, expectedDbHits = 40, expectedPageCacheHits = 10, expectedPageCacheMisses = 28) + assertRecorded(profile, pipe3.id, expectedRows = 1, expectedDbHits = 2, expectedPageCacheHits = 25, expectedPageCacheMisses = 33) } test("should count stuff going through Apply multiple times") { @@ -131,21 +119,15 @@ class ProfilerTest extends CypherFunSuite { val rhs = ProfilerTestPipe(s2, "rhs", rows = 20, dbAccess = 30)(idGen.id()) val apply = ApplyPipe(lhs, rhs)(idGen.id()) val queryContext: QueryContext = prepareQueryContext() - val profiler = new Profiler + val profile = new InterpretedProfileInformation + val profiler = new Profiler(DatabaseInfo.COMMUNITY, profile) val queryState = QueryStateHelper.emptyWith(query = queryContext, decorator = profiler) - val planDescription = createPlanDescription( - "start1" -> s1, - "start2" -> s2, - "lhs" -> lhs, - "rhs" -> rhs, - "apply" -> apply) // WHEN we create the results, materialize(apply.createResults(queryState)) - val decoratedResult = profiler.decorate(() => planDescription, verifyProfileReady = () => {}) // THEN - assertRecorded(decoratedResult(), "rhs", expectedRows = 10 * 20, expectedDbHits = 10 * 30) + assertRecorded(profile, rhs.id, expectedRows = 10 * 20, expectedDbHits = 10 * 30) } test("count dbhits for NestedPipes") { @@ -160,20 +142,15 @@ class ProfilerTest extends CypherFunSuite { val pipeUnderInspection = ProjectionPipe(start2, Map("x" -> innerPipe))(idGen.id()) val queryContext: QueryContext = prepareQueryContext() - val profiler = new Profiler + val profile = new InterpretedProfileInformation + val profiler = new Profiler(DatabaseInfo.COMMUNITY, profile) val queryState = QueryStateHelper.emptyWith(query = queryContext, decorator = profiler) - val planDescription = createPlanDescription( - "start1" -> start1, - "start2" -> start2, - "lhs" -> testPipe, - "Projection" -> pipeUnderInspection) // WHEN we create the results, materialize(pipeUnderInspection.createResults(queryState)) - val decoratedResult = profiler.decorate(() => planDescription, verifyProfileReady = () => {}) // THEN the ProjectionNewPipe has correctly recorded the dbhits - assertRecorded(decoratedResult(), "Projection", expectedRows = 1, expectedDbHits = DB_HITS) + assertRecorded(profile, pipeUnderInspection.id, expectedRows = 1, expectedDbHits = DB_HITS) } test("count page cache hits for NestedPipes") { @@ -188,20 +165,15 @@ class ProfilerTest extends CypherFunSuite { val pipeUnderInspection = ProjectionPipe(start2, Map("x" -> innerPipe))(idGen.id()) val queryContext: QueryContext = prepareQueryContext(statisticProvider) - val profiler = new Profiler(DatabaseInfo.HA) + val profile = new InterpretedProfileInformation + val profiler = new Profiler(DatabaseInfo.HA, profile) val queryState = QueryStateHelper.emptyWith(query = queryContext, decorator = profiler) - val planDescription = createPlanDescription( - "start1" -> start1, - "start2" -> start2, - "lhs" -> testPipe, - "Projection" -> pipeUnderInspection) // WHEN we create the results, materialize(pipeUnderInspection.createResults(queryState)) - val decoratedResult = profiler.decorate(() => planDescription, verifyProfileReady = () => {}) // THEN the ProjectionNewPipe has correctly recorded the page cache hits - assertRecorded(decoratedResult(), "Projection", expectedRows = 1, expectedDbHits = 2, expectedPageCacheHits = 3, expectedPageCacheMisses = 4) + assertRecorded(profile, pipeUnderInspection.id, expectedRows = 1, expectedDbHits = 2, expectedPageCacheHits = 3, expectedPageCacheMisses = 4) } test("count dbhits for deeply nested NestedPipes") { @@ -220,28 +192,19 @@ class ProfilerTest extends CypherFunSuite { val pipeUnderInspection = ProjectionPipe(start3, Map("x" -> pipeExpression))(idGen.id()) val queryContext: QueryContext = prepareQueryContext() - val profiler = new Profiler + val profile = new InterpretedProfileInformation + val profiler = new Profiler(DatabaseInfo.COMMUNITY, profile) val queryState = QueryStateHelper.emptyWith(query = queryContext, decorator = profiler) // WHEN we create the results, materialize(pipeUnderInspection.createResults(queryState)) - val description = createPlanDescription( - "start1" -> start1, - "start2" -> start2, - "start3" -> start3, - "profiler1" -> profiler1, - "profiler2" -> profiler2, - "innerInner" -> innerInnerPipe, - "Projection" -> pipeUnderInspection - ) - val decoratedResult = profiler.decorate(() => description, verifyProfileReady = () => {}) // THEN the ProjectionNewPipe has correctly recorded the dbhits - assertRecorded(decoratedResult(), "Projection", expectedRows = 1, expectedDbHits = DB_HITS * 2) + assertRecorded(profile, pipeUnderInspection.id, expectedRows = 1, expectedDbHits = DB_HITS * 2) } test("should not count rows multiple times when the same pipe is used multiple times") { - val profiler = new Profiler + val profiler = new Profiler(DatabaseInfo.COMMUNITY, new InterpretedProfileInformation) val pipe1 = ArgumentPipe()(idGen.id()) val ctx1: QueryContext = prepareQueryContext() @@ -266,7 +229,7 @@ class ProfilerTest extends CypherFunSuite { } test("should not count dbhits multiple times when the same pipe is used multiple times") { - val profiler = new Profiler + val profiler = new Profiler(DatabaseInfo.COMMUNITY, new InterpretedProfileInformation) val pipe1 = ArgumentPipe()(idGen.id()) val ctx1: QueryContext = prepareQueryContext() @@ -294,22 +257,17 @@ class ProfilerTest extends CypherFunSuite { queryContext } - private def assertRecorded(result: InternalPlanDescription, name: String, expectedRows: Int, expectedDbHits: Int, - expectedPageCacheHits: Int = 0, expectedPageCacheMisses: Int = 0) { - val pipeArgs: Seq[Argument] = result.find(name).flatMap(_.arguments) - pipeArgs shouldNot be(empty) - pipeArgs.collect { - case DbHits(count) => withClue("DbHits:")(count should equal(expectedDbHits)) - } - pipeArgs.collect { - case Rows(seenRows) => withClue("Rows:")(seenRows should equal(expectedRows)) - } - pipeArgs.collect { - case PageCacheHits(seenHits) => withClue("PageCacheHits:")(seenHits should equal(expectedPageCacheHits)) - } - pipeArgs.collect { - case PageCacheMisses(seenMisses) => withClue("PageCacheMisses:")(seenMisses should equal(expectedPageCacheMisses)) - } + private def assertRecorded(result: QueryProfile, + id: Id, + expectedRows: Int, + expectedDbHits: Int, + expectedPageCacheHits: Int = 0, + expectedPageCacheMisses: Int = 0) { + val data: OperatorProfile = result.operatorProfile(id.x) + withClue("DbHits:")(data.dbHits() should equal(expectedDbHits)) + withClue("Rows:")(data.rows should equal(expectedRows)) + withClue("PageCacheHits:")(data.pageCacheHits() should equal(expectedPageCacheHits)) + withClue("PageCacheMisses:")(data.pageCacheMisses() should equal(expectedPageCacheMisses)) } private def materialize(iterator: Iterator[_]) { diff --git a/community/cypher/interpreted-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/interpreted/DelegatingQueryContext.scala b/community/cypher/interpreted-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/interpreted/DelegatingQueryContext.scala index 4507e019030b0..eb76ac76adcd9 100644 --- a/community/cypher/interpreted-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/interpreted/DelegatingQueryContext.scala +++ b/community/cypher/interpreted-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/interpreted/DelegatingQueryContext.scala @@ -279,14 +279,11 @@ abstract class DelegatingQueryContext(val inner: QueryContext) extends QueryCont allowed: Array[String]): UserDefinedAggregator = singleDbHit(inner.aggregateFunction(name, allowed)) - override def isGraphKernelResultValue(v: Any): Boolean = - inner.isGraphKernelResultValue(v) - override def detachDeleteNode(node: Long): Int = manyDbHits(inner.detachDeleteNode(node)) override def assertSchemaWritesAllowed(): Unit = inner.assertSchemaWritesAllowed() - override def asObject(value: AnyValue): Any = inner.asObject(value) + override def asObject(value: AnyValue): AnyRef = inner.asObject(value) } class DelegatingOperations[T](protected val inner: Operations[T]) extends Operations[T] { diff --git a/community/cypher/interpreted-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/interpreted/TransactionBoundQueryContext.scala b/community/cypher/interpreted-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/interpreted/TransactionBoundQueryContext.scala index 1fb780455af84..8a1443d8d73b9 100644 --- a/community/cypher/interpreted-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/interpreted/TransactionBoundQueryContext.scala +++ b/community/cypher/interpreted-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/interpreted/TransactionBoundQueryContext.scala @@ -470,7 +470,7 @@ sealed class TransactionBoundQueryContext(val transactionalContext: Transactiona else cursor.isDense } - override def asObject(value: AnyValue): Any = value.map(valueMapper) + override def asObject(value: AnyValue): AnyRef = value.map(valueMapper) class NodeOperations extends BaseOperations[NodeValue] { @@ -1029,8 +1029,6 @@ sealed class TransactionBoundQueryContext(val transactionalContext: Transactiona } } - override def isGraphKernelResultValue(v: Any): Boolean = v.isInstanceOf[PropertyContainer] || v.isInstanceOf[Path] - private def buildPathFinder(depth: Int, expander: Expander, pathPredicate: KernelPredicate[Path], filters: Seq[KernelPredicate[PropertyContainer]]): ShortestPath = { val startExpander = expander match { diff --git a/community/cypher/interpreted-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/interpreted/pipes/ConstraintOperationPipe.scala b/community/cypher/interpreted-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/interpreted/pipes/ConstraintOperationPipe.scala deleted file mode 100644 index 6353a5c3d2f2b..0000000000000 --- a/community/cypher/interpreted-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/interpreted/pipes/ConstraintOperationPipe.scala +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2002-2018 "Neo4j," - * Neo4j Sweden AB [http://neo4j.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.cypher.internal.runtime.interpreted.pipes - -import org.neo4j.cypher.internal.runtime.interpreted.ExecutionContext -import org.neo4j.cypher.internal.runtime.interpreted.commands._ -import org.neo4j.cypher.internal.runtime.interpreted.commands.values.KeyToken -import org.neo4j.cypher.internal.planner.v3_5.spi.IndexDescriptor -import org.opencypher.v9_0.util.attribution.Id - -class ConstraintOperationPipe(op: PropertyConstraintOperation, keyToken: KeyToken, propertyKey: KeyToken) - (val id: Id = Id.INVALID_ID) extends Pipe { - protected def internalCreateResults(state: QueryState): Iterator[ExecutionContext] = { - val keyTokenId = keyToken.getOrCreateId(state.query) - val propertyKeyId = propertyKey.getOrCreateId(state.query) - - op match { - case _: CreateUniqueConstraint => state.query.createUniqueConstraint(IndexDescriptor(keyTokenId, propertyKeyId)) - case _: DropUniqueConstraint => state.query.dropUniqueConstraint(IndexDescriptor(keyTokenId, propertyKeyId)) - case _: CreateNodePropertyExistenceConstraint => state.query.createNodePropertyExistenceConstraint(keyTokenId, propertyKeyId) - case _: DropNodePropertyExistenceConstraint => state.query.dropNodePropertyExistenceConstraint(keyTokenId, propertyKeyId) - case _: CreateRelationshipPropertyExistenceConstraint => state.query.createRelationshipPropertyExistenceConstraint(keyTokenId, propertyKeyId) - case _: DropRelationshipPropertyExistenceConstraint => state.query.dropRelationshipPropertyExistenceConstraint(keyTokenId, propertyKeyId) - } - - Iterator.empty - } -} diff --git a/community/cypher/interpreted-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/interpreted/pipes/IndexOperationPipe.scala b/community/cypher/interpreted-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/interpreted/pipes/IndexOperationPipe.scala deleted file mode 100644 index 26acfbb7b54df..0000000000000 --- a/community/cypher/interpreted-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/interpreted/pipes/IndexOperationPipe.scala +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2002-2018 "Neo4j," - * Neo4j Sweden AB [http://neo4j.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.cypher.internal.runtime.interpreted.pipes - -import org.neo4j.cypher.internal.planner.v3_5.spi.IndexDescriptor -import org.neo4j.cypher.internal.runtime.interpreted.ExecutionContext -import org.neo4j.cypher.internal.runtime.interpreted.commands.{CreateIndex, DropIndex, IndexOperation} -import org.opencypher.v9_0.util.attribution.Id -import org.opencypher.v9_0.util.{LabelId, PropertyKeyId, SyntaxException} - -case class IndexOperationPipe(indexOp: IndexOperation)(val id: Id = Id.INVALID_ID) extends Pipe { - protected def internalCreateResults(state: QueryState): Iterator[ExecutionContext] = { - val queryContext = state.query - - val labelId = queryContext.getOrCreateLabelId(indexOp.label) - - indexOp match { - case CreateIndex(_, propertyKeys, _) => - val propertyKeyIds: Seq[Int] = propertyKeys.map( queryContext.getOrCreatePropertyKeyId ) - queryContext.addIndexRule(IndexDescriptor(LabelId(labelId), propertyKeyIds.map(PropertyKeyId))) - - case DropIndex(_, propertyKeys, _) => - val propertyKeyIds: Seq[Int] = propertyKeys.map( queryContext.getOrCreatePropertyKeyId ) - queryContext.dropIndexRule(IndexDescriptor(LabelId(labelId), propertyKeyIds.map(PropertyKeyId))) - - case _ => - throw new UnsupportedOperationException("Unknown IndexOperation encountered") - } - - Iterator.empty - } - - private def single[T](s: Seq[T]): T = { - if (s.isEmpty || s.tail.nonEmpty) - throw new SyntaxException("Cypher support only one property key per index right now") - s(0) - } -} diff --git a/community/cypher/interpreted-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/interpreted/pipes/PipeDecorator.scala b/community/cypher/interpreted-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/interpreted/pipes/PipeDecorator.scala index 7e7c5a653fcb8..8db62f0d9b24e 100644 --- a/community/cypher/interpreted-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/interpreted/pipes/PipeDecorator.scala +++ b/community/cypher/interpreted-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/interpreted/pipes/PipeDecorator.scala @@ -20,7 +20,6 @@ package org.neo4j.cypher.internal.runtime.interpreted.pipes import org.neo4j.cypher.internal.runtime.interpreted.ExecutionContext -import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription /* A PipeDecorator is used to instrument calls between Pipes, and between a Pipe and the graph @@ -30,8 +29,6 @@ trait PipeDecorator { def decorate(pipe: Pipe, iter: Iterator[ExecutionContext]): Iterator[ExecutionContext] - def decorate(plan: () => InternalPlanDescription, verifyProfileReady: () => Unit): () => InternalPlanDescription - /* * Returns the inner decorator of this decorator. The inner decorator is used for nested expressions * where the `decorate` should refer to the parent pipe instead of the calling pipe. @@ -42,8 +39,6 @@ trait PipeDecorator { object NullPipeDecorator extends PipeDecorator { def decorate(pipe: Pipe, iter: Iterator[ExecutionContext]): Iterator[ExecutionContext] = iter - def decorate(plan: () => InternalPlanDescription, verifyProfileReady: () => Unit): () => InternalPlanDescription = plan - def decorate(pipe: Pipe, state: QueryState): QueryState = state def innerDecorator(pipe: Pipe): PipeDecorator = NullPipeDecorator diff --git a/community/cypher/interpreted-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/interpreted/pipes/PipeExecutionBuilderContext.scala b/community/cypher/interpreted-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/interpreted/pipes/PipeExecutionBuilderContext.scala index ed3bc4d8ebba3..d75dd0fd3df84 100644 --- a/community/cypher/interpreted-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/interpreted/pipes/PipeExecutionBuilderContext.scala +++ b/community/cypher/interpreted-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/interpreted/pipes/PipeExecutionBuilderContext.scala @@ -20,8 +20,6 @@ package org.neo4j.cypher.internal.runtime.interpreted.pipes import org.opencypher.v9_0.ast.semantics.SemanticTable -import org.neo4j.cypher.internal.planner.v3_5.spi.PlanningAttributes.Cardinalities case class PipeExecutionBuilderContext(semanticTable: SemanticTable, - readOnly: Boolean, - cardinalities: Cardinalities) + readOnly: Boolean) diff --git a/community/cypher/interpreted-runtime/src/test/scala/org/neo4j/cypher/internal/runtime/interpreted/QueryContextAdaptation.scala b/community/cypher/interpreted-runtime/src/test/scala/org/neo4j/cypher/internal/runtime/interpreted/QueryContextAdaptation.scala index 9b459c0c005bf..8c9d93c48c7c3 100644 --- a/community/cypher/interpreted-runtime/src/test/scala/org/neo4j/cypher/internal/runtime/interpreted/QueryContextAdaptation.scala +++ b/community/cypher/interpreted-runtime/src/test/scala/org/neo4j/cypher/internal/runtime/interpreted/QueryContextAdaptation.scala @@ -113,9 +113,6 @@ trait QueryContextAdaptation { override def dropNodeKeyConstraint(descriptor: IndexDescriptor): Unit = ??? - // Check if a runtime value is a node, relationship, path or some such value returned from - override def isGraphKernelResultValue(v: Any): Boolean = ??? - override def transactionalContext: QueryTransactionalContext = ??? override def allShortestPath(left: Long, right: Long, depth: Int, expander: Expander, pathPredicate: KernelPredicate[Path], filters: Seq[KernelPredicate[PropertyContainer]]): scala.Iterator[Path] = ??? diff --git a/community/cypher/interpreted-runtime/src/test/scala/org/neo4j/cypher/internal/runtime/interpreted/pipes/ProcedureCallPipeTest.scala b/community/cypher/interpreted-runtime/src/test/scala/org/neo4j/cypher/internal/runtime/interpreted/pipes/ProcedureCallPipeTest.scala index 946ee03f18f44..b04ad63965c3c 100644 --- a/community/cypher/interpreted-runtime/src/test/scala/org/neo4j/cypher/internal/runtime/interpreted/pipes/ProcedureCallPipeTest.scala +++ b/community/cypher/interpreted-runtime/src/test/scala/org/neo4j/cypher/internal/runtime/interpreted/pipes/ProcedureCallPipeTest.scala @@ -117,8 +117,6 @@ class ProcedureCallPipeTest class FakeQueryContext(id: Int, result: Seq[Any] => Iterator[Array[AnyRef]], expectedAccessMode: ProcedureAccessMode) extends QueryContext with QueryContextAdaptation { - override def isGraphKernelResultValue(v: Any): Boolean = false - override def callReadOnlyProcedure(id: Int, args: Seq[Any], allowed: Array[String]) = { expectedAccessMode should equal(ProcedureReadOnlyAccess(emptyStringArray)) doIt(id, args, allowed) diff --git a/community/cypher/runtime-util/src/main/java/org/neo4j/cypher/result/OperatorProfile.java b/community/cypher/runtime-util/src/main/java/org/neo4j/cypher/result/OperatorProfile.java new file mode 100644 index 0000000000000..d5daab3a93d53 --- /dev/null +++ b/community/cypher/runtime-util/src/main/java/org/neo4j/cypher/result/OperatorProfile.java @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2002-2018 "Neo4j," + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.cypher.result; + +import org.neo4j.helpers.MathUtil; + +/** + * Profile for a operator during a query execution. + */ +public interface OperatorProfile +{ + /** + * Time spent executing this operator. + */ + long time(); + + /** + * Database hits caused while executing this operator. This is an approximate measure + * of how many nodes, records and properties that have been read. + */ + long dbHits(); + + /** + * Number of rows produced by this operator. + */ + long rows(); + + /** + * Page cache page hits while executing this operator. + */ + long pageCacheHits(); + + /** + * Page cache page misses while executing this operator. + */ + long pageCacheMisses(); + + default double pageCacheHitRatio() + { + return MathUtil.portion( pageCacheHits(), pageCacheMisses() ); + } + + OperatorProfile NONE = new OperatorProfile() + { + @Override + public long time() + { + return -1; + } + + @Override + public long dbHits() + { + return -1; + } + + @Override + public long rows() + { + return -1; + } + + @Override + public long pageCacheHits() + { + return -1; + } + + @Override + public long pageCacheMisses() + { + return -1; + } + }; +} diff --git a/community/cypher/runtime-util/src/main/java/org/neo4j/cypher/result/QueryProfile.java b/community/cypher/runtime-util/src/main/java/org/neo4j/cypher/result/QueryProfile.java new file mode 100644 index 0000000000000..66d1c05477677 --- /dev/null +++ b/community/cypher/runtime-util/src/main/java/org/neo4j/cypher/result/QueryProfile.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2002-2018 "Neo4j," + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.cypher.result; + +/** + * Profile of a query execution. + */ +public interface QueryProfile +{ + /** + * Get profile for specific operator. + * + * Note: the operator should really be type as an [[org.opencypher.v9_0.util.attribution.Id]], + * but this is not possible because of a bug with scala `AnyVal`s and inheritance. + * + * See https://github.com/lampepfl/dotty/issues/1169 for a discussion of the same issue in Dotty. + * + * @param operatorId operator id + */ + OperatorProfile operatorProfile( int operatorId ); + + QueryProfile NONE = operatorId -> OperatorProfile.NONE; +} diff --git a/community/cypher/runtime-util/src/main/java/org/neo4j/cypher/result/RuntimeResult.java b/community/cypher/runtime-util/src/main/java/org/neo4j/cypher/result/RuntimeResult.java new file mode 100644 index 0000000000000..978a44fe21751 --- /dev/null +++ b/community/cypher/runtime-util/src/main/java/org/neo4j/cypher/result/RuntimeResult.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2002-2018 "Neo4j," + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.cypher.result; + +import org.neo4j.cypher.internal.runtime.QueryStatistics; +import org.neo4j.graphdb.ResourceIterator; + +/** + * The result API of a Cypher runtime + */ +public interface RuntimeResult extends AutoCloseable +{ + String[] fieldNames(); + + boolean isIterable(); + + ResourceIterator> asIterator(); + + boolean isExhausted(); + + void accept( QueryResult.QueryResultVisitor visitor ) + throws E; + + QueryStatistics queryStatistics(); + + QueryProfile queryProfile(); + + @Override + void close(); +} diff --git a/community/cypher/runtime-util/src/main/scala/org/neo4j/cypher/internal/runtime/InternalExecutionResult.scala b/community/cypher/runtime-util/src/main/scala/org/neo4j/cypher/internal/runtime/InternalExecutionResult.scala index 2b9617982e9bf..73990aae57624 100644 --- a/community/cypher/runtime-util/src/main/scala/org/neo4j/cypher/internal/runtime/InternalExecutionResult.scala +++ b/community/cypher/runtime-util/src/main/scala/org/neo4j/cypher/internal/runtime/InternalExecutionResult.scala @@ -25,41 +25,33 @@ import java.lang import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription import org.neo4j.cypher.result.QueryResult import org.neo4j.graphdb.Result.ResultVisitor -import org.neo4j.graphdb._ +import org.neo4j.graphdb.{Notification, QueryExecutionType, ResourceIterator} import scala.collection.JavaConverters._ -trait InternalExecutionResult extends Iterator[Map[String, Any]] with QueryResult { - - def columns: List[String] = List(fieldNames():_*) - def columnAs[T](column: String): Iterator[T] +trait InternalExecutionResult extends QueryResult { def javaColumns: java.util.List[String] = java.util.Arrays.asList(fieldNames():_*) def javaColumnAs[T](column: String): ResourceIterator[T] - def javaIterator: ResourceIterator[java.util.Map[String, Any]] + def javaIterator: ResourceIterator[java.util.Map[String, AnyRef]] def dumpToString(writer: PrintWriter) def dumpToString(): String - def queryStatistics(): QueryStatistics + override def queryStatistics(): QueryStatistics - def planDescriptionRequested: Boolean - def executionPlanDescription(): InternalPlanDescription + def executionMode: ExecutionMode - def executionPlanString(): String = executionPlanDescription().toString + def executionPlanDescription(): InternalPlanDescription def queryType: InternalQueryType - def executionMode: ExecutionMode - def notifications: Iterable[Notification] override def getNotifications: lang.Iterable[Notification] = notifications.asJava def accept[E <: Exception](visitor: ResultVisitor[E]): Unit - def withNotifications(notification: Notification*): InternalExecutionResult - def executionType: QueryExecutionType = { val qt = queryType match { @@ -76,4 +68,20 @@ trait InternalExecutionResult extends Iterator[Map[String, Any]] with QueryResul case NormalMode => QueryExecutionType.query(qt) } } + + def close(success: Boolean): Unit = close() + + def closeOnError(t: Throwable): Throwable = { + try { + close(success = false) + } catch { + case thrownDuringClose: Throwable => + try { + t.addSuppressed(thrownDuringClose) + } catch { + case _: Throwable => // Ignore + } + } + t + } } diff --git a/community/cypher/runtime-util/src/main/scala/org/neo4j/cypher/internal/runtime/QueryContext.scala b/community/cypher/runtime-util/src/main/scala/org/neo4j/cypher/internal/runtime/QueryContext.scala index 03bb7ec57cd59..ab9e8aa1d1823 100644 --- a/community/cypher/runtime-util/src/main/scala/org/neo4j/cypher/internal/runtime/QueryContext.scala +++ b/community/cypher/runtime-util/src/main/scala/org/neo4j/cypher/internal/runtime/QueryContext.scala @@ -168,7 +168,7 @@ trait QueryContext extends TokenContext with DbAccess { def nodeIsDense(node: Long): Boolean - def asObject(value: AnyValue): Any + def asObject(value: AnyValue): AnyRef // Legacy dependency between kernel and compiler def variableLengthPathExpand(realNode: Long, minHops: Option[Int], maxHops: Option[Int], direction: SemanticDirection, relTypes: Seq[String]): Iterator[Path] @@ -205,10 +205,6 @@ trait QueryContext extends TokenContext with DbAccess { def aggregateFunction(id: Int, allowed: Array[String]): UserDefinedAggregator def aggregateFunction(name: QualifiedName, allowed: Array[String]): UserDefinedAggregator - // Check if a runtime value is a node, relationship, path or some such value returned from - // other query context values by calling down to the underlying database - def isGraphKernelResultValue(v: Any): Boolean - def detachDeleteNode(id: Long): Int def assertSchemaWritesAllowed(): Unit diff --git a/community/cypher/runtime-util/src/main/scala/org/neo4j/cypher/internal/runtime/QueryStatistics.scala b/community/cypher/runtime-util/src/main/scala/org/neo4j/cypher/internal/runtime/QueryStatistics.scala index 879aae03a9d5a..4e791b8e909f8 100644 --- a/community/cypher/runtime-util/src/main/scala/org/neo4j/cypher/internal/runtime/QueryStatistics.scala +++ b/community/cypher/runtime-util/src/main/scala/org/neo4j/cypher/internal/runtime/QueryStatistics.scala @@ -52,7 +52,7 @@ case class QueryStatistics(@BeanProperty nodesCreated: Int = 0, @BeanProperty val constraintsRemoved: Int = uniqueConstraintsRemoved + existenceConstraintsRemoved + nodekeyConstraintsRemoved - def containsUpdates: Boolean = + override def containsUpdates: Boolean = nodesCreated > 0 || relationshipsCreated > 0 || propertiesSet > 0 || diff --git a/community/cypher/runtime-util/src/main/scala/org/neo4j/cypher/internal/runtime/planDescription/LogicalPlan2PlanDescription.scala b/community/cypher/runtime-util/src/main/scala/org/neo4j/cypher/internal/runtime/planDescription/LogicalPlan2PlanDescription.scala index 05a01293d34fa..27a6c010f552f 100644 --- a/community/cypher/runtime-util/src/main/scala/org/neo4j/cypher/internal/runtime/planDescription/LogicalPlan2PlanDescription.scala +++ b/community/cypher/runtime-util/src/main/scala/org/neo4j/cypher/internal/runtime/planDescription/LogicalPlan2PlanDescription.scala @@ -105,6 +105,10 @@ case class LogicalPlan2PlanDescription(readOnly: Boolean, cardinalities: Cardina val signature = Signature(call.qualifiedName, call.callArguments, call.callResultTypes) PlanDescriptionImpl(id, "ProcedureCall", NoChildren, Seq(signature), variables) + case StandAloneProcedureCall(signature, args, resultSymbols, callResultIndices) => + val signatureDesc = Signature(signature.name, Seq.empty, resultSymbols) + PlanDescriptionImpl(id, "ProcedureCall", NoChildren, Seq(signatureDesc), resultSymbols.map(_._1).toSet) + case RelationshipCountFromCountStore(ident, startLabel, typeNames, endLabel, _) => val exp = CountRelationshipsExpression(ident, startLabel.map(_.name), typeNames.map(_.name), endLabel.map(_.name)) @@ -113,10 +117,43 @@ case class LogicalPlan2PlanDescription(readOnly: Boolean, cardinalities: Cardina case _: UndirectedRelationshipByIdSeek => PlanDescriptionImpl(id, "UndirectedRelationshipByIdSeek", NoChildren, Seq.empty, variables) + case _: CreateIndex => + PlanDescriptionImpl(id, "CreateIndex", NoChildren, Seq.empty, variables) + + case _: DropIndex => + PlanDescriptionImpl(id, "DropIndex", NoChildren, Seq.empty, variables) + + case _: CreateUniquePropertyConstraint => + PlanDescriptionImpl(id, "CreateUniquePropertyConstraint", NoChildren, Seq.empty, variables) + + case _: CreateNodeKeyConstraint => + PlanDescriptionImpl(id, "CreateNodeKeyConstraint", NoChildren, Seq.empty, variables) + + case _: CreateNodePropertyExistenceConstraint => + PlanDescriptionImpl(id, "CreateNodePropertyExistenceConstraint", NoChildren, Seq.empty, variables) + + case _: CreateRelationshipPropertyExistenceConstraint => + PlanDescriptionImpl(id, "CreateRelationshipPropertyExistenceConstraint", NoChildren, Seq.empty, variables) + + case _: DropUniquePropertyConstraint => + PlanDescriptionImpl(id, "DropUniquePropertyConstraint", NoChildren, Seq.empty, variables) + + case _: DropNodeKeyConstraint => + PlanDescriptionImpl(id, "DropNodeKeyConstraint", NoChildren, Seq.empty, variables) + + case _: DropNodePropertyExistenceConstraint => + PlanDescriptionImpl(id, "DropNodePropertyExistenceConstraint", NoChildren, Seq.empty, variables) + + case _: DropRelationshipPropertyExistenceConstraint => + PlanDescriptionImpl(id, "DropRelationshipPropertyExistenceConstraint", NoChildren, Seq.empty, variables) + case x => throw new InternalException(s"Unknown plan type: ${x.getClass.getSimpleName}. Missing a case?") } - result.addArgument(EstimatedRows(cardinalities.get(plan.id).amount)) + if (cardinalities.isDefinedAt(plan.id)) + result.addArgument(EstimatedRows(cardinalities.get(plan.id).amount)) + else + result } override protected def build(plan: LogicalPlan, source: InternalPlanDescription): InternalPlanDescription = { diff --git a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/BuiltInProcedureAcceptanceTest.scala b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/BuiltInProcedureAcceptanceTest.scala index e550ff98f89ab..3472551b72e2e 100644 --- a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/BuiltInProcedureAcceptanceTest.scala +++ b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/BuiltInProcedureAcceptanceTest.scala @@ -58,8 +58,9 @@ class BuiltInProcedureAcceptanceTest extends ProcedureCallAcceptanceTest with Cy relate(d1, neo, "PART_OF", "Hallo") // When - val query = "CALL db.schema()" - val result = executeWith(Configs.Procs, query).toList + // we cannot assert on the results because on each call + // the generated virtual nodes will have different IDs + val result = executeWith(Configs.Procs, "CALL db.schema()", expectedDifferentResults = Configs.Procs).toList // Then result.size should equal(1) @@ -305,7 +306,7 @@ class BuiltInProcedureAcceptanceTest extends ProcedureCallAcceptanceTest with Cy "status" -> "index created")) ) - executeWith(Configs.Procs, "CALL db.awaitIndexes(10)") + graph.execute("CALL db.awaitIndexes(10)") // when val listResult = executeWith(Configs.Procs, "CALL db.indexes()") @@ -336,7 +337,7 @@ class BuiltInProcedureAcceptanceTest extends ProcedureCallAcceptanceTest with Cy "status" -> "uniqueness constraint online")) ) - executeWith(Configs.Procs, "CALL db.awaitIndexes(10)") + graph.execute("CALL db.awaitIndexes(10)") // when val listResult = executeWith(Configs.Procs, "CALL db.indexes()") @@ -366,7 +367,7 @@ class BuiltInProcedureAcceptanceTest extends ProcedureCallAcceptanceTest with Cy "status" -> "node key constraint online")) ) - executeWith(Configs.Procs, "CALL db.awaitIndexes(10)") + graph.execute("CALL db.awaitIndexes(10)") // when val listResult = executeWith(Configs.Procs, "CALL db.indexes()") diff --git a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/CypherComparisonSupport.scala b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/CypherComparisonSupport.scala index 596f93d383dbc..35d9ef3bab7f3 100644 --- a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/CypherComparisonSupport.scala +++ b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/CypherComparisonSupport.scala @@ -24,15 +24,12 @@ package org.neo4j.internal.cypher.acceptance import org.neo4j.cypher._ import org.neo4j.cypher.internal.RewindableExecutionResult -import org.neo4j.cypher.internal.compiler.v3_1.{CartesianPoint => CartesianPointv3_1, GeographicPoint => GeographicPointv3_1} -import org.neo4j.cypher.internal.runtime.InternalExecutionResult -import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription +import org.neo4j.cypher.internal.runtime.planDescription.{Argument, InternalPlanDescription} import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription.Arguments.{Planner => IPDPlanner, PlannerVersion => IPDPlannerVersion, Runtime => IPDRuntime, RuntimeVersion => IPDRuntimeVersion} import org.neo4j.graphdb.Result import org.neo4j.graphdb.config.Setting import org.neo4j.graphdb.factory.GraphDatabaseSettings import org.neo4j.test.{TestEnterpriseGraphDatabaseFactory, TestGraphDatabaseFactory} -import org.neo4j.values.storable.{CoordinateReferenceSystem, Values} import org.opencypher.v9_0.util.Eagerly import org.opencypher.v9_0.util.test_helpers.CypherTestSupport import org.scalatest.Assertions @@ -54,7 +51,7 @@ trait CypherComparisonSupport extends CypherTestSupport { /** * Get rid of Arrays and java.util.Map to make it easier to compare results by equality. */ - implicit class RichInternalExecutionResults(res: InternalExecutionResult) { + implicit class RichInternalExecutionResults(res: RewindableExecutionResult) { def toComparableResultWithOptions(replaceNaNs: Boolean): Seq[Map[String, Any]] = res.toList.toComparableSeq(replaceNaNs) def toComparableResult: Seq[Map[String, Any]] = res.toList.toComparableSeq(replaceNaNs = false) @@ -68,8 +65,6 @@ trait CypherComparisonSupport extends CypherTestSupport { def toComparableSeq(replaceNaNs: Boolean): Seq[Map[String, Any]] = { def convert(v: Any): Any = v match { - case p: GeographicPointv3_1 => Values.pointValue(CoordinateReferenceSystem.get(p.crs.url), p.longitude, p.latitude) - case p: CartesianPointv3_1 => Values.pointValue(CoordinateReferenceSystem.get(p.crs.url), p.x, p.y) case a: Array[_] => a.toList.map(convert) case m: Map[_, _] => Eagerly.immutableMapValues(m, convert) @@ -104,9 +99,9 @@ trait CypherComparisonSupport extends CypherTestSupport { for (thisScenario <- scenariosToExecute) { val expectedToFailWithSpecificMessage = expectedSpecificFailureFromEffective.containsScenario(thisScenario) - val tryResult: Try[InternalExecutionResult] = Try(innerExecute(s"CYPHER ${thisScenario.preparserOptions} $query", params)) + val tryResult: Try[RewindableExecutionResult] = Try(innerExecute(s"CYPHER ${thisScenario.preparserOptions} $query", params)) tryResult match { - case (Success(_)) => + case Success(_) => if (expectedToFailWithSpecificMessage) { fail("Unexpectedly Succeeded in " + thisScenario.name) } @@ -134,14 +129,53 @@ trait CypherComparisonSupport extends CypherTestSupport { possibleErrors == Seq.empty || (actualError != null && possibleErrors.exists(s => actualError.replaceAll("\\r", "").contains(s.replaceAll("\\r", "")))) } + protected def dumpToString(expectSucceed: TestConfiguration, + query: String, + params: Map[String, Any] = Map.empty): String = { + + val paramValue = asMapValue(params) + + case class DumpResult(maybeResult:Try[String], scenario: TestScenario) + + val results: Seq[DumpResult] = + Configs.AbsolutelyAll.scenarios.toSeq.map { + scenario => { + val queryText = s"CYPHER ${scenario.preparserOptions} $query" + val txContext = graph.transactionalContext(query = queryText -> params) + val maybeResult = + Try(eengine.execute(queryText, paramValue, txContext).resultAsString()) + DumpResult(maybeResult, scenario) + } + } + + val (corrects, incorrects) = results.partition(t => expectSucceed.containsScenario(t.scenario)) + val reference = corrects.head.maybeResult.get + for (correct <- corrects) { + withClue(s"Failed with scenario '${correct.scenario.preparserOptions}'") { + correct.maybeResult.get should equal(reference) + } + } + for (incorrect <- incorrects) { + withClue(s"Unexpectedly succeeded with scenario '${incorrect.scenario.preparserOptions}'") { + incorrect.maybeResult match { + case Success(result) => + result should not equal reference + case Failure(error) => + // expected + } + } + } + reference + } + protected def executeWith(expectSucceed: TestConfiguration, query: String, expectedDifferentResults: TestConfiguration = Configs.Empty, planComparisonStrategy: PlanComparisonStrategy = DoNotComparePlans, - resultAssertionInTx: Option[InternalExecutionResult => Unit] = None, + resultAssertionInTx: Option[RewindableExecutionResult => Unit] = None, executeBefore: () => Unit = () => {}, executeExpectedFailures: Boolean = true, - params: Map[String, Any] = Map.empty): InternalExecutionResult = { + params: Map[String, Any] = Map.empty): RewindableExecutionResult = { // Never consider Morsel even if test requests it val expectSucceedEffective = expectSucceed - Configs.Morsel @@ -222,7 +256,7 @@ trait CypherComparisonSupport extends CypherTestSupport { expectedToSucceed: Boolean, executeBefore: () => Unit, params: Map[String, Any], - resultAssertionInTx: Option[InternalExecutionResult => Unit], + resultAssertionInTx: Option[RewindableExecutionResult => Unit], executeExpectedFailures: Boolean, rollback: Boolean = true) = { @@ -262,10 +296,10 @@ trait CypherComparisonSupport extends CypherTestSupport { } @deprecated("Rewrite to use executeWith instead") - protected def assertResultsSameDeprecated(result1: InternalExecutionResult, result2: InternalExecutionResult, queryText: String, errorMsg: String, replaceNaNs: Boolean = false): Unit = + protected def assertResultsSameDeprecated(result1: RewindableExecutionResult, result2: RewindableExecutionResult, queryText: String, errorMsg: String, replaceNaNs: Boolean = false): Unit = assertResultsSame(result1, result2, queryText, errorMsg, replaceNaNs) - private def assertResultsSame(result1: InternalExecutionResult, result2: InternalExecutionResult, queryText: String, errorMsg: String, replaceNaNs: Boolean = false): Unit = { + private def assertResultsSame(result1: RewindableExecutionResult, result2: RewindableExecutionResult, queryText: String, errorMsg: String, replaceNaNs: Boolean = false): Unit = { withClue(errorMsg) { if (queryText.toLowerCase contains "order by") { result1.toComparableResultWithOptions(replaceNaNs) should contain theSameElementsInOrderAs result2.toComparableResultWithOptions(replaceNaNs) @@ -275,7 +309,7 @@ trait CypherComparisonSupport extends CypherTestSupport { } } - private def assertResultsNotSame(result1: InternalExecutionResult, result2: InternalExecutionResult, queryText: String, errorMsg: String, replaceNaNs: Boolean = false): Unit = { + private def assertResultsNotSame(result1: RewindableExecutionResult, result2: RewindableExecutionResult, queryText: String, errorMsg: String, replaceNaNs: Boolean = false): Unit = { withClue(errorMsg) { if (queryText.toLowerCase contains "order by") { result1.toComparableResultWithOptions(replaceNaNs) shouldNot contain theSameElementsInOrderAs result2.toComparableResultWithOptions(replaceNaNs) @@ -285,19 +319,19 @@ trait CypherComparisonSupport extends CypherTestSupport { } } - // Should this really be deprecated? We have real use cases where we want to get the InternalExecutionResult + // Should this really be deprecated? We have real use cases where we want to get the RewindableExecutionResult // But do NOT want comparison support, for example see the query statistics support used in CompositeNodeKeyAcceptanceTests @deprecated("Rewrite to use executeWith instead") - protected def innerExecuteDeprecated(queryText: String, params: Map[String, Any] = Map.empty): InternalExecutionResult = + protected def innerExecuteDeprecated(queryText: String, params: Map[String, Any] = Map.empty): RewindableExecutionResult = innerExecute(queryText, params) - private def innerExecute(queryText: String, params: Map[String, Any]): InternalExecutionResult = { + private def innerExecute(queryText: String, params: Map[String, Any]): RewindableExecutionResult = { val innerResult: Result = eengine.execute(queryText, asMapValue(params), graph.transactionalContext(query = queryText -> params)) RewindableExecutionResult(innerResult) } - def evaluateTo(expected: Seq[Map[String, Any]]): Matcher[InternalExecutionResult] = new Matcher[InternalExecutionResult] { - override def apply(actual: InternalExecutionResult): MatchResult = { + def evaluateTo(expected: Seq[Map[String, Any]]): Matcher[RewindableExecutionResult] = new Matcher[RewindableExecutionResult] { + override def apply(actual: RewindableExecutionResult): MatchResult = { MatchResult( matches = actual.toComparableResult == expected.toComparableSeq(replaceNaNs = false), rawFailureMessage = s"Results differ: ${actual.toComparableResult} did not equal to $expected", @@ -440,17 +474,17 @@ object CypherComparisonSupport { sealed trait PlanComparisonStrategy extends Assertions { - def compare(expectSucceed: TestConfiguration, scenario: TestScenario, result: InternalExecutionResult): Unit + def compare(expectSucceed: TestConfiguration, scenario: TestScenario, result: RewindableExecutionResult): Unit } case object DoNotComparePlans extends PlanComparisonStrategy { - override def compare(expectSucceed: TestConfiguration, scenario: TestScenario, result: InternalExecutionResult): Unit = {} + override def compare(expectSucceed: TestConfiguration, scenario: TestScenario, result: RewindableExecutionResult): Unit = {} } - case class ComparePlansWithPredicate(predicate: (InternalPlanDescription) => Boolean, + case class ComparePlansWithPredicate(predicate: InternalPlanDescription => Boolean, expectPlansToFailPredicate: TestConfiguration = TestConfiguration.empty, predicateFailureMessage: String = "") extends PlanComparisonStrategy { - override def compare(expectSucceed: TestConfiguration, scenario: TestScenario, result: InternalExecutionResult): Unit = { + override def compare(expectSucceed: TestConfiguration, scenario: TestScenario, result: RewindableExecutionResult): Unit = { val comparePlans = expectSucceed - expectPlansToFailPredicate if (comparePlans.containsScenario(scenario)) { if (!predicate(result.executionPlanDescription())) { @@ -464,9 +498,9 @@ object CypherComparisonSupport { } } - case class ComparePlansWithAssertion(assertion: (InternalPlanDescription) => Unit, + case class ComparePlansWithAssertion(assertion: InternalPlanDescription => Unit, expectPlansToFail: TestConfiguration = TestConfiguration.empty) extends PlanComparisonStrategy { - override def compare(expectSucceed: TestConfiguration, scenario: TestScenario, result: InternalExecutionResult): Unit = { + override def compare(expectSucceed: TestConfiguration, scenario: TestScenario, result: RewindableExecutionResult): Unit = { val comparePlans = expectSucceed - expectPlansToFail if (comparePlans.containsScenario(scenario)) { withClue(s"plan for ${scenario.name}\n") { @@ -502,7 +536,7 @@ object CypherComparisonSupport { def preparserOptions: String = List(version.name, planner.preparserOption, runtime.preparserOption).mkString(" ") - def checkResultForSuccess(query: String, internalExecutionResult: InternalExecutionResult): Unit = { + def checkResultForSuccess(query: String, internalExecutionResult: RewindableExecutionResult): Unit = { val (reportedRuntime: String, reportedPlanner: String, reportedVersion: String, reportedPlannerVersion: String) = extractConfiguration(internalExecutionResult) if (!runtime.acceptedRuntimeNames.contains(reportedRuntime)) fail(s"did not use ${runtime.acceptedRuntimeNames} runtime - instead $reportedRuntime was used. Scenario $name") @@ -514,7 +548,7 @@ object CypherComparisonSupport { fail(s"did not use ${version.acceptedPlannerVersionNames} planner version - instead $reportedPlannerVersion was used. Scenario $name") } - def checkResultForFailure(query: String, internalExecutionResult: Try[InternalExecutionResult]): Unit = { + def checkResultForFailure(query: String, internalExecutionResult: Try[RewindableExecutionResult]): Unit = { internalExecutionResult match { case Failure(_) => // not unexpected case Success(result) => @@ -528,8 +562,10 @@ object CypherComparisonSupport { } } - private def extractConfiguration(result: InternalExecutionResult): (String, String, String, String) = { - val arguments = result.executionPlanDescription().arguments + private def extractConfiguration(result: RewindableExecutionResult): (String, String, String, String) = + extractConfiguration(result.executionPlanDescription().arguments) + + private def extractConfiguration(arguments: Seq[Argument]): (String, String, String, String) = { val reportedRuntime = arguments.collectFirst { case IPDRuntime(reported) => reported } @@ -591,6 +627,8 @@ object CypherComparisonSupport { object Configs { + def Default: TestConfiguration = TestConfiguration(Versions.Default, Planners.Default, Runtimes.Default) + def Compiled: TestConfiguration = TestConfiguration(Versions.v3_5, Planners.Cost, Runtimes(Runtimes.CompiledSource, Runtimes.CompiledBytecode)) def Morsel: TestConfiguration = TestConfiguration(Versions.Default, Planners.Default, Runtimes(Runtimes.Morsel)) @@ -639,7 +677,14 @@ object CypherComparisonSupport { def BackwardsCompatibility: TestConfiguration = TestConfiguration(Versions.V2_3 -> Versions.V3_1, Planners.all, Runtimes.Default) + TestScenario(Versions.V3_4, Planners.Cost, Runtimes.Default) - def Procs: TestConfiguration = TestScenario(Versions.Default, Planners.Default, Runtimes.ProcedureOrSchema) + def DefaultProcs: TestConfiguration = TestScenario(Versions.Default, Planners.Default, Runtimes.ProcedureOrSchema) + + def Procs: TestConfiguration = + TestConfiguration( + Versions(Versions.Default, Versions.V3_4, Versions.v3_5), + Planners(Planners.Default, Planners.Cost), + Runtimes(Runtimes.Default, Runtimes.ProcedureOrSchema) + ) /** * Handy configs for things not supported in older versions @@ -657,7 +702,7 @@ object CypherComparisonSupport { If you are unsure what you need, this is a good start. It's not really all scenarios, but this is testing all interesting scenarios. */ - def All: TestConfiguration = AbsolutelyAll - Procs + def All: TestConfiguration = AbsolutelyAll - DefaultProcs /** * These are all configurations that will be executed even if not explicitly expected to succeed or fail. diff --git a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/DumpToStringAcceptanceTest.scala b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/DumpToStringAcceptanceTest.scala index 0e80c079c3332..fa59e947a46f4 100644 --- a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/DumpToStringAcceptanceTest.scala +++ b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/DumpToStringAcceptanceTest.scala @@ -33,7 +33,7 @@ class DumpToStringAcceptanceTest extends ExecutionEngineFunSuite with CypherComp test("format node") { createNode(Map("prop1" -> "A", "prop2" -> 2)) - executeWith(Configs.All + Configs.Morsel, ("match (n) return n")).dumpToString() should + dumpToString(Configs.AbsolutelyAll + Configs.Morsel, "match (n) return n") should equal( """+----------------------------+ || n | @@ -47,7 +47,7 @@ class DumpToStringAcceptanceTest extends ExecutionEngineFunSuite with CypherComp test("format relationship") { relate(createNode(), createNode(), "T", Map("prop1" -> "A", "prop2" -> 2)) - executeWith(Configs.All + Configs.Morsel, "match ()-[r]->() return r").dumpToString() should equal( + dumpToString(Configs.AbsolutelyAll + Configs.Morsel, "match ()-[r]->() return r") should equal( """+--------------------------+ || r | |+--------------------------+ @@ -58,7 +58,7 @@ class DumpToStringAcceptanceTest extends ExecutionEngineFunSuite with CypherComp } test("format collection of maps") { - executeWith(Configs.All + Configs.Morsel, """RETURN [{ inner: 'Map1' }, { inner: 'Map2' }]""").dumpToString() should + dumpToString(Configs.AbsolutelyAll + Configs.Morsel, """RETURN [{ inner: 'Map1' }, { inner: 'Map2' }]""") should equal( """+----------------------------------------+ || [{ inner: 'Map1' }, { inner: 'Map2' }] | diff --git a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/EagerizationAcceptanceTest.scala b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/EagerizationAcceptanceTest.scala index 521a37dc7b7b8..ed088dca2f09f 100644 --- a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/EagerizationAcceptanceTest.scala +++ b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/EagerizationAcceptanceTest.scala @@ -1345,7 +1345,7 @@ class EagerizationAcceptanceTest createNode() val query = "MATCH (m1:Two), (m2:Two), (n) MERGE (q) ON MATCH SET q:Two RETURN count(*) AS c" - val result: InternalExecutionResult = executeWith(Configs.UpdateConf, query, + val result = executeWith(Configs.UpdateConf, query, planComparisonStrategy = testEagerPlanComparisonStrategy(1)) assertStats(result, labelsAdded = 1) result.toList should equal(List(Map("c" -> 36))) @@ -1357,7 +1357,7 @@ class EagerizationAcceptanceTest createNode() val query = "MATCH (m1:Two), (m2:Two), (n) MERGE (q:Three) ON MATCH SET q:Two RETURN count(*) AS c" - val result: InternalExecutionResult = executeWith(Configs.UpdateConf, query, + val result = executeWith(Configs.UpdateConf, query, planComparisonStrategy = testEagerPlanComparisonStrategy(1)) assertStats(result, labelsAdded = 2, nodesCreated = 1) result.toList should equal(List(Map("c" -> 12))) @@ -1369,7 +1369,7 @@ class EagerizationAcceptanceTest createNode() val query = "MATCH (a:Two), (b) MERGE (q {p: 1}) RETURN count(*) AS c" - val result: InternalExecutionResult = executeWith(Configs.UpdateConf, query, + val result = executeWith(Configs.UpdateConf, query, planComparisonStrategy = testEagerPlanComparisonStrategy(1)) assertStats(result, nodesCreated = 1, propertiesWritten = 1) result.toList should equal(List(Map("c" -> 6))) @@ -1381,7 +1381,7 @@ class EagerizationAcceptanceTest createNode() val query = "MATCH (m1:Two), (m2:Two) MERGE (q) ON MATCH SET q:One RETURN count(*) AS c" - val result: InternalExecutionResult = executeWith(Configs.UpdateConf, query, + val result = executeWith(Configs.UpdateConf, query, planComparisonStrategy = testEagerPlanComparisonStrategy(0, Configs.Rule2_3)) assertStats(result, labelsAdded = 3) result.toList should equal(List(Map("c" -> 12))) @@ -1393,7 +1393,7 @@ class EagerizationAcceptanceTest createNode() val query = "MATCH (m1:Two), (m2:Two), (n) MERGE (q) ON CREATE SET q:Two RETURN count(*) AS c" - val result: InternalExecutionResult = executeWith(Configs.UpdateConf, query, + val result = executeWith(Configs.UpdateConf, query, planComparisonStrategy = testEagerPlanComparisonStrategy(1)) assertStats(result, labelsAdded = 0) result.toList should equal(List(Map("c" -> 36))) @@ -1823,7 +1823,7 @@ class EagerizationAcceptanceTest createNode() val query = "MATCH (m1:Two), (m2:Two), (n) SET n:Two RETURN count(*) AS c" - val result: InternalExecutionResult = executeWith(Configs.UpdateConf, query, + val result = executeWith(Configs.UpdateConf, query, planComparisonStrategy = testEagerPlanComparisonStrategy(1)) assertStats(result, labelsAdded = 1) result.toList should equal(List(Map("c" -> 12))) diff --git a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/ExecutionEngineTest.scala b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/ExecutionEngineTest.scala index b93c236e8ce3d..191cc3725b052 100644 --- a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/ExecutionEngineTest.scala +++ b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/ExecutionEngineTest.scala @@ -130,10 +130,9 @@ class ExecutionEngineTest extends ExecutionEngineFunSuite with QueryStatisticsTe relate(n1, n2, "KNOWS") relate(n1, n3, "KNOWS") - val result = executeWith(Configs.All + Configs.Morsel, + dumpToString(Configs.AbsolutelyAll + Configs.Morsel, s"match (node)-[rel:KNOWS]->(x) where id(node) = ${n1.getId} return x, node" ) - result.dumpToString() } test("should Find Nodes By Exact Index Lookup") { @@ -406,7 +405,7 @@ order by a.COL1""".format(a, b)) test("shouldToStringArraysPrettily") { createNode("foo" -> Array("one", "two")) - val string = executeWith(Configs.All + Configs.Morsel, """match (n) where id(n) = 0 return n.foo""").dumpToString() + val string = dumpToString(Configs.AbsolutelyAll + Configs.Morsel, """match (n) where id(n) = 0 return n.foo""") string should include("""["one","two"]""") } @@ -477,8 +476,8 @@ order by a.COL1""".format(a, b)) test("shouldReturnASimplePath") { val errorMessage = List("Index `missingIndex` does not exist") - val conf = startConf + Configs.Procs - val conf2 = Configs.AllRulePlanners + Configs.DefaultInterpreted + Configs.Procs + val conf = startConf + Configs.DefaultProcs + val conf2 = Configs.AllRulePlanners + Configs.DefaultInterpreted + Configs.DefaultProcs failWithError(conf, "start a=node:missingIndex(key='value') return a", errorMessage) failWithError(conf, "start a=node:missingIndex('value') return a", errorMessage) failWithError(conf2, "start a=relationship:missingIndex(key='value') return a", errorMessage) @@ -725,9 +724,15 @@ order by a.COL1""".format(a, b)) val labelName = "Person" val propertyKeys = Seq("name") - val testconfiguration = TestConfiguration(Versions(V3_1, v3_5, Versions.Default), Planners.Default, Runtimes(ProcedureOrSchema, Runtimes.Default)) + Configs.Rule2_3 + val testConfiguration = + TestConfiguration( + Versions(V3_1, v3_5, Versions.Default), + Planners.Default, + Runtimes(ProcedureOrSchema, Runtimes.Default) + ) + Configs.Rule2_3 + Configs.Cost3_4 + // WHEN - executeWith(testconfiguration, s"""CREATE INDEX ON :$labelName(${propertyKeys.reduce(_ ++ "," ++ _)})""") + executeWith(testConfiguration, s"""CREATE INDEX ON :$labelName(${propertyKeys.reduce(_ ++ "," ++ _)})""") // THEN graph.inTx { diff --git a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/ExplainAcceptanceTest.scala b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/ExplainAcceptanceTest.scala index f8b5616486c27..162e906f900cd 100644 --- a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/ExplainAcceptanceTest.scala +++ b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/ExplainAcceptanceTest.scala @@ -23,6 +23,7 @@ package org.neo4j.internal.cypher.acceptance import org.neo4j.cypher.ExecutionEngineFunSuite +import org.neo4j.cypher.internal.runtime.{ExplainMode, NormalMode} import org.neo4j.internal.cypher.acceptance.CypherComparisonSupport.Configs class ExplainAcceptanceTest extends ExecutionEngineFunSuite with CypherComparisonSupport { @@ -31,7 +32,7 @@ class ExplainAcceptanceTest extends ExecutionEngineFunSuite with CypherCompariso createNode() val result = executeWith(Configs.All + Configs.Morsel, "match (n) return n") - result.planDescriptionRequested should equal(false) + result.executionMode should equal(NormalMode) result shouldNot be(empty) } @@ -39,7 +40,7 @@ class ExplainAcceptanceTest extends ExecutionEngineFunSuite with CypherCompariso createNode() val result = executeWith(Configs.All + Configs.Morsel, "explain match (n) return n") - result.planDescriptionRequested should equal(true) + result.executionMode should equal(ExplainMode) result should be(empty) } @@ -73,7 +74,6 @@ class ExplainAcceptanceTest extends ExecutionEngineFunSuite with CypherCompariso val result = executeWith(Configs.Interpreted, query) val plan = result.executionPlanDescription().toString - result.close() plan.toString should include("NestedPlanExpression(VarExpand-Argument)") } diff --git a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/LoadCsvAcceptanceTest.scala b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/LoadCsvAcceptanceTest.scala index 511ff55619dcb..399e015c49b94 100644 --- a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/LoadCsvAcceptanceTest.scala +++ b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/LoadCsvAcceptanceTest.scala @@ -44,7 +44,7 @@ class LoadCsvAcceptanceTest extends ExecutionEngineFunSuite with BeforeAndAfterAll with QueryStatisticsTestSupport with CreateTempFileTestSupport with CypherComparisonSupport with RunWithConfigTestSupport { - val expectedToFail = Configs.AbsolutelyAll - Configs.Compiled - Configs.Cost2_3 + private val expectedToFail = Configs.AbsolutelyAll - Configs.Compiled - Configs.Cost2_3 def csvUrls(f: PrintWriter => Unit) = Seq( createCSVTempFileURL(f), @@ -68,7 +68,8 @@ class LoadCsvAcceptanceTest | CREATE (order:Order{orderID: row.OrderId}) | CREATE (user)-[acc:ORDERED]->(order) | RETURN count(*)""".stripMargin - ) + ).resultAsString() + graph.createIndex("User", "userID") // when & then @@ -134,8 +135,6 @@ class LoadCsvAcceptanceTest val result = execute(s"LOAD CSV FROM '$filePathForQuery' AS line CREATE (a {name: line[0]}) RETURN a.name") assertStats(result, nodesCreated = 1, propertiesWritten = 1) - result.close() - assert(Files.deleteIfExists(path)) } @@ -583,7 +582,6 @@ class LoadCsvAcceptanceTest val result = executeWith(Configs.UpdateConf, query) result.columnAs("c").toList should equal(List(0)) - result.close() } test("empty headers file should not throw") { diff --git a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/LoadCsvWithQuotesAcceptanceTest.scala b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/LoadCsvWithQuotesAcceptanceTest.scala index d73e6dd6d503c..3a36be8946c12 100644 --- a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/LoadCsvWithQuotesAcceptanceTest.scala +++ b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/LoadCsvWithQuotesAcceptanceTest.scala @@ -130,7 +130,7 @@ class LoadCsvWithQuotesAcceptanceTest extends ExecutionEngineFunSuite with RunWi } } - def executeWithCustomDb(db: GraphDatabaseCypherService, query: String): InternalExecutionResult = { + def executeWithCustomDb(db: GraphDatabaseCypherService, query: String): RewindableExecutionResult = { val engine = ExecutionEngineHelper.createEngine(db) RewindableExecutionResult(engine.execute(query, VirtualValues.emptyMap(), diff --git a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/MatchAcceptanceTest.scala b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/MatchAcceptanceTest.scala index 69156e2ebba9a..3bd166ef8ccd4 100644 --- a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/MatchAcceptanceTest.scala +++ b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/MatchAcceptanceTest.scala @@ -156,8 +156,6 @@ class MatchAcceptanceTest extends ExecutionEngineFunSuite with QueryStatisticsTe val result = executeWith(Configs.Interpreted, query) result.size should be(0) - result.hasNext should be(false) - } // Not TCK material -- only one integer type @@ -589,8 +587,8 @@ class MatchAcceptanceTest extends ExecutionEngineFunSuite with QueryStatisticsTe |RETURN project.p""".stripMargin //WHEN - val first = executeWith(Configs.UpdateConf, query).length - val second = executeWith(Configs.UpdateConf, query).length + val first = executeWith(Configs.UpdateConf, query).size + val second = executeWith(Configs.UpdateConf, query).size val check = executeWith(Configs.All, "MATCH (f:Folder) RETURN f.name").toSet //THEN @@ -624,8 +622,8 @@ class MatchAcceptanceTest extends ExecutionEngineFunSuite with QueryStatisticsTe //WHEN - val first = executeWith(Configs.UpdateConf, query).length - val second = executeWith(Configs.UpdateConf, query).length + val first = executeWith(Configs.UpdateConf, query).size + val second = executeWith(Configs.UpdateConf, query).size val check = executeWith(Configs.All, "MATCH (f:Folder) RETURN f.name").toSet //THEN diff --git a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/MatchAggregationsBackedByCountStoreAcceptanceTest.scala b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/MatchAggregationsBackedByCountStoreAcceptanceTest.scala index 73b8d1540f0d4..6b7a9ce281af1 100644 --- a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/MatchAggregationsBackedByCountStoreAcceptanceTest.scala +++ b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/MatchAggregationsBackedByCountStoreAcceptanceTest.scala @@ -22,6 +22,7 @@ */ package org.neo4j.internal.cypher.acceptance +import org.neo4j.cypher.internal.RewindableExecutionResult import org.neo4j.cypher.internal.runtime.InternalExecutionResult import org.neo4j.cypher.{ExecutionEngineFunSuite, QueryPlanTestSupport, QueryStatisticsTestSupport} import org.neo4j.internal.cypher.acceptance.CypherComparisonSupport.{ComparePlansWithAssertion, Configs, TestConfiguration} @@ -415,7 +416,7 @@ class MatchAggregationsBackedByCountStoreAcceptanceTest setupBigModel(label2 = "Admin") - val resultAssertionInTx = Some({ result: InternalExecutionResult => result.toList should equal(List(Map("userKnows" -> 2, "otherKnows" -> 1))) }) + val resultAssertionInTx = Some({ result: RewindableExecutionResult => result.toList should equal(List(Map("userKnows" -> 2, "otherKnows" -> 1))) }) val result = executeWith( expectSucceed, query, @@ -682,7 +683,7 @@ class MatchAggregationsBackedByCountStoreAcceptanceTest val resultAssertionInTx = if (assertCountInTransaction) { - Some({ result: InternalExecutionResult => result.columnAs(result.columns.head).toSet[Any] should equal(Set(expectedCount)) }) + Some({ result: RewindableExecutionResult => result.columnAs(result.columns.head).toSet[Any] should equal(Set(expectedCount)) }) } else { None } diff --git a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/MergeIntoPlanningAcceptanceTest.scala b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/MergeIntoPlanningAcceptanceTest.scala index fb7398ab1112e..d99db2e1106a4 100644 --- a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/MergeIntoPlanningAcceptanceTest.scala +++ b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/MergeIntoPlanningAcceptanceTest.scala @@ -23,7 +23,6 @@ package org.neo4j.internal.cypher.acceptance import org.neo4j.cypher.ExecutionEngineFunSuite -import org.neo4j.cypher.internal.runtime.InternalExecutionResult class MergeIntoPlanningAcceptanceTest extends ExecutionEngineFunSuite{ @@ -34,7 +33,7 @@ class MergeIntoPlanningAcceptanceTest extends ExecutionEngineFunSuite{ //when val update = execute("""MATCH (a {name:'A'}), (b {name:'B'}) - |MERGE (a)-[r:TYPE]->(b) ON CREATE SET r.name = 'foo'""".stripMargin) + |MERGE (a)-[r:TYPE]->(b) ON CREATE SET r.name = 'foo'""".stripMargin) //then update.executionPlanDescription() should includeSomewhere.aPlan("Expand(Into)") @@ -100,7 +99,4 @@ class MergeIntoPlanningAcceptanceTest extends ExecutionEngineFunSuite{ //then update.executionPlanDescription() should includeSomewhere.aPlan("Expand(Into)") } - - //MERGE INTO is only used by the rule planner - override def execute(q: String, params: (String, Any)*): InternalExecutionResult= super.execute(s"$q", params:_*) } diff --git a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/NotificationAcceptanceTest.scala b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/NotificationAcceptanceTest.scala index b86f7b8281a46..a6b29aead0e69 100644 --- a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/NotificationAcceptanceTest.scala +++ b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/NotificationAcceptanceTest.scala @@ -644,14 +644,14 @@ class NotificationAcceptanceTest extends ExecutionEngineFunSuite with CypherComp val query = "EXPLAIN CYPHER runtime=slotted START n=node(0) RETURN n" val result = innerExecuteDeprecated(query, Map.empty) val notifications = result.notifications - notifications should contain(RUNTIME_UNSUPPORTED.notification(new graphdb.InputPosition(31,1,32))) + notifications should contain(RUNTIME_UNSUPPORTED.notification(graphdb.InputPosition.empty)) } test("should warn when using CREATE UNIQUE in newer runtimes") { val query = "EXPLAIN CYPHER runtime=slotted MATCH (root { name: 'root' }) CREATE UNIQUE (root)-[:LOVES]-(someone) RETURN someone" val result = innerExecuteDeprecated(query, Map.empty) val notifications = result.notifications - notifications should contain(RUNTIME_UNSUPPORTED.notification(new graphdb.InputPosition(61,1,62))) + notifications should contain(RUNTIME_UNSUPPORTED.notification(graphdb.InputPosition.empty)) } test("should warn when using contains on an index with SLOW_CONTAINS limitation") { diff --git a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/ParameterValuesAcceptanceTest.scala b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/ParameterValuesAcceptanceTest.scala index f23401d367a45..b74a2c5cb226c 100644 --- a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/ParameterValuesAcceptanceTest.scala +++ b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/ParameterValuesAcceptanceTest.scala @@ -32,7 +32,7 @@ class ParameterValuesAcceptanceTest extends ExecutionEngineFunSuite with CypherC // given val node = createLabeledNode("Person") val result = executeWith(Configs.All + Configs.Morsel, "WITH {param} as p RETURN p", params = Map("param" -> Array(node))) - val outputP = result.next.get("p").get + val outputP = result.head("p") outputP should equal(Array(node)) } @@ -45,7 +45,7 @@ class ParameterValuesAcceptanceTest extends ExecutionEngineFunSuite with CypherC | RETURN ANY(n IN collect(distinct node) WHERE n IN nodes1) as exists """.stripMargin val r = executeWith(Configs.Interpreted - Configs.Version2_3, query) - r.next().apply("exists") should equal(false) + r.toList should equal(List(Map("exists" -> false))) } test("should not erase the type of an empty array sent as parameter") { diff --git a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/PatternExpressionImplementationAcceptanceTest.scala b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/PatternExpressionImplementationAcceptanceTest.scala index 97158c8d5f730..714c775ad55b9 100644 --- a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/PatternExpressionImplementationAcceptanceTest.scala +++ b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/PatternExpressionImplementationAcceptanceTest.scala @@ -80,7 +80,7 @@ class PatternExpressionImplementationAcceptanceTest extends ExecutionEngineFunSu val rel4 = relate(start2, d) val result = executeWith(Configs.Interpreted, "match (n) return case when n:A then (n)-->(:C) when n:B then (n)-->(:D) else 42 end as p") - .map(_.mapValues { + .toList.map(_.mapValues { case l: Seq[Any] => l.toSet case x => x }).toSet @@ -132,7 +132,7 @@ class PatternExpressionImplementationAcceptanceTest extends ExecutionEngineFunSu val rel4 = relate(start2, d) val result = executeWith(Configs.Interpreted, "match (n) with case when n:A then (n)-->(:C) when n:B then (n)-->(:D) else 42 end as p, count(n) as c return p, c") - .map(_.mapValues { + .toList.map(_.mapValues { case l: Seq[Any] => l.toSet case x => x }).toSet diff --git a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/PeriodicCommitAcceptanceTest.scala b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/PeriodicCommitAcceptanceTest.scala index 618972ed86c68..a5ba72a700cc1 100644 --- a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/PeriodicCommitAcceptanceTest.scala +++ b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/PeriodicCommitAcceptanceTest.scala @@ -89,7 +89,6 @@ class PeriodicCommitAcceptanceTest extends ExecutionEngineFunSuite arguments.find( _.isInstanceOf[PageCacheMisses]) shouldBe defined result.columnAs[Long]("id").toList should equal(List("1","2","3","4","5")) val afterTxId = txIdStore.getLastClosedTransactionId - result.close() afterTxId should equal(beforeTxId + 5) } diff --git a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/ProfilerAcceptanceTest.scala b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/ProfilerAcceptanceTest.scala index 4ade62b13d266..ca664410c2033 100644 --- a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/ProfilerAcceptanceTest.scala +++ b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/ProfilerAcceptanceTest.scala @@ -22,11 +22,16 @@ */ package org.neo4j.internal.cypher.acceptance +import org.neo4j.cypher.internal.RewindableExecutionResult import org.neo4j.cypher.internal.planner.v3_5.spi.GraphStatistics import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription.Arguments.{DbHits, Rows} import org.neo4j.cypher.internal.runtime.planDescription.{Argument, InternalPlanDescription} import org.neo4j.cypher.internal.runtime.{CreateTempFileTestSupport, InternalExecutionResult} import org.neo4j.cypher.internal.runtime.{CreateTempFileTestSupport, ProfileMode} +import org.neo4j.cypher.internal.runtime.{CreateTempFileTestSupport, ProfileMode} +import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription.Arguments.{DbHits, EstimatedRows, Rows, Signature} +import org.neo4j.cypher.internal.runtime.planDescription.{Argument, InternalPlanDescription} +import org.neo4j.cypher.internal.v3_5.logical.plans.QualifiedName import org.neo4j.cypher.{ExecutionEngineFunSuite, ProfilerStatisticsNotReadyException, TxCounts} import org.neo4j.graphdb.QueryExecutionException import org.neo4j.internal.cypher.acceptance.CypherComparisonSupport.{Configs, TestConfiguration} @@ -340,7 +345,7 @@ class ProfilerAcceptanceTest extends ExecutionEngineFunSuite with CreateTempFile test("should support profiling optional match and with") { createLabeledNode(Map("x" -> 1), "Label") - val executionResult: InternalExecutionResult = profileWithExecute(Configs.Interpreted, "match (n) optional match (n)--(m) with n, m where m is null return n.x as A") + val executionResult = profileWithExecute(Configs.Interpreted, "match (n) optional match (n)--(m) with n, m where m is null return n.x as A") val result = executionResult.toList.head result("A") should equal(1) } @@ -651,11 +656,11 @@ class ProfilerAcceptanceTest extends ExecutionEngineFunSuite with CreateTempFile result.executionPlanDescription() should includeSomewhere.aPlan("Filter").withDBHits(14) } - type Planner = (String, Map[String, Any]) => InternalExecutionResult + type Planner = (String, Map[String, Any]) => RewindableExecutionResult - def profileWithPlanner(planner: Planner, q: String, params: Map[String, Any]): InternalExecutionResult = { + def profileWithPlanner(planner: Planner, q: String, params: Map[String, Any]): RewindableExecutionResult = { val result = planner("profile " + q, params) - assert(result.planDescriptionRequested, "result not marked with planDescriptionRequested") + result.executionMode should equal(ProfileMode) val planDescription: InternalPlanDescription = result.executionPlanDescription() planDescription.flatten.foreach { @@ -668,9 +673,9 @@ class ProfilerAcceptanceTest extends ExecutionEngineFunSuite with CreateTempFile result } - def profileWithExecute(configuration: TestConfiguration, q: String): InternalExecutionResult = { + def profileWithExecute(configuration: TestConfiguration, q: String): RewindableExecutionResult = { val result = executeWith(configuration, "profile " + q) - assert(result.planDescriptionRequested, "result not marked with planDescriptionRequested") + result.executionMode should equal(ProfileMode) val planDescription: InternalPlanDescription = result.executionPlanDescription() planDescription.flatten.foreach { @@ -683,9 +688,9 @@ class ProfilerAcceptanceTest extends ExecutionEngineFunSuite with CreateTempFile result } - override def profile(q: String, params: (String, Any)*): InternalExecutionResult = fail("Don't use profile all together in ProfilerAcceptanceTest") + override def profile(q: String, params: (String, Any)*): RewindableExecutionResult = fail("Don't use profile all together in ProfilerAcceptanceTest") - def legacyProfile(q: String, params: (String, Any)*): InternalExecutionResult = profileWithPlanner(innerExecuteDeprecated, q, params.toMap) + def legacyProfile(q: String, params: (String, Any)*): RewindableExecutionResult = profileWithPlanner(innerExecuteDeprecated, q, params.toMap) private def getArgument[A <: Argument](plan: InternalPlanDescription)(implicit manifest: ClassTag[A]): A = plan.arguments.collectFirst { case x: A => x diff --git a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/SameQueryStressTest.scala b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/SameQueryStressTest.scala index a2585e53e3b7d..e975e229b606a 100644 --- a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/SameQueryStressTest.scala +++ b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/SameQueryStressTest.scala @@ -41,8 +41,8 @@ class SameQueryStressTest extends ExecutionEngineFunSuite { test(s"concurrent query execution in $runtime") { // Given - execute(TestGraph.movies) - val expected = execute(lookup).dumpToString() + graph.execute(TestGraph.movies) + val expected = graph.execute(lookup).resultAsString() // When val nThreads = 10 @@ -52,7 +52,7 @@ class SameQueryStressTest extends ExecutionEngineFunSuite { executor.submit(new Callable[Array[String]] { override def call(): Array[String] = { (for (j <- 1 to 1000) yield { - execute(lookup).dumpToString() + graph.execute(lookup).resultAsString() }).toArray } }) diff --git a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/SerializationAcceptanceTest.scala b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/SerializationAcceptanceTest.scala index 913be4da75396..bb8846bf2c7b2 100644 --- a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/SerializationAcceptanceTest.scala +++ b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/SerializationAcceptanceTest.scala @@ -24,7 +24,7 @@ package org.neo4j.internal.cypher.acceptance import org.neo4j.cypher._ -class SerializationAcceptanceTest extends ExecutionEngineFunSuite with QueryStatisticsTestSupport { +class SerializationAcceptanceTest extends ExecutionEngineFunSuite { // serialization of deleted entities @@ -34,9 +34,9 @@ class SerializationAcceptanceTest extends ExecutionEngineFunSuite with QueryStat val query = "MATCH (n) DELETE n RETURN n" graph.inTx { - val result = execute(query) + val result = graph.execute(query).resultAsString() - result.dumpToString() should include("Node[0]{deleted}") + result should include("Node[0]{deleted}") } } @@ -46,9 +46,9 @@ class SerializationAcceptanceTest extends ExecutionEngineFunSuite with QueryStat val query = "MATCH (n) RETURN n" graph.inTx { - val result = execute(query) + val result = graph.execute(query).resultAsString() - result.dumpToString() should not include "deleted" + result should not include "deleted" } } @@ -58,9 +58,9 @@ class SerializationAcceptanceTest extends ExecutionEngineFunSuite with QueryStat val query = "MATCH ()-[r]->() RETURN r" graph.inTx { - val result = execute(query) + val result = graph.execute(query).resultAsString() - result.dumpToString() should not include "deleted" + result should not include "deleted" } } @@ -70,9 +70,9 @@ class SerializationAcceptanceTest extends ExecutionEngineFunSuite with QueryStat val query = "MATCH ()-[r]->() DELETE r RETURN r" graph.inTx { - val result = execute(query) + val result = graph.execute(query).resultAsString() - result.dumpToString() should include(":T[0]{deleted}") + result should include(":T[0]{deleted}") } } @@ -82,11 +82,11 @@ class SerializationAcceptanceTest extends ExecutionEngineFunSuite with QueryStat val query = "MATCH (a)-[r]->(b) DELETE a, r, b RETURN *" graph.inTx { - val result = execute(query) + val result = graph.execute(query).resultAsString() - result.dumpToString() should include(":T[0]{deleted}") - result.dumpToString() should include("Node[0]{deleted}") - result.dumpToString() should include("Node[1]{deleted}") + result should include(":T[0]{deleted}") + result should include("Node[0]{deleted}") + result should include("Node[1]{deleted}") } } @@ -96,11 +96,11 @@ class SerializationAcceptanceTest extends ExecutionEngineFunSuite with QueryStat val query = "MATCH p=(a)-[r]->(b) DELETE p RETURN p" graph.inTx { - val result = execute(query) + val result = graph.execute(query).resultAsString() - result.dumpToString() should include(":T[0]{deleted}") - result.dumpToString() should include("Node[0]{deleted}") - result.dumpToString() should include("Node[1]{deleted}") + result should include("[0:T,deleted]") + result should include("(0,deleted)") + result should include("(1,deleted)") } } @@ -110,11 +110,11 @@ class SerializationAcceptanceTest extends ExecutionEngineFunSuite with QueryStat val query = "MATCH p=(a)-[r]->(b) DELETE a, r RETURN p" graph.inTx { - val result = execute(query) + val result = graph.execute(query).resultAsString() - result.dumpToString() should include(":T[0]{deleted}") - result.dumpToString() should include("Node[0]{deleted}") - result.dumpToString() should not include("Node[1]{deleted}") + result should include("[0:T,deleted]") + result should include("(0,deleted)") + result should not include "(1,deleted)" } } @@ -124,11 +124,11 @@ class SerializationAcceptanceTest extends ExecutionEngineFunSuite with QueryStat val query = "MATCH p=(a)-[r]->(b) DELETE r RETURN p" graph.inTx { - val result = execute(query) + val result = graph.execute(query).resultAsString() - result.dumpToString() should include(":T[0]{deleted}") - result.dumpToString() should not include("Node[0]{deleted}") - result.dumpToString() should not include("Node[1]{deleted}") + result should include("[0:T,deleted]") + result should not include "(0,deleted)" + result should not include "(1,deleted)" } } diff --git a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/SetAcceptanceTest.scala b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/SetAcceptanceTest.scala index 6ee4eb2db3b3c..8b0ff5d27f317 100644 --- a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/SetAcceptanceTest.scala +++ b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/SetAcceptanceTest.scala @@ -54,7 +54,7 @@ class SetAcceptanceTest extends ExecutionEngineFunSuite with QueryStatisticsTest // when executeWith(Configs.UpdateConf, "MATCH (n) SET n.prop = tofloat(n.prop)") - executeWith(Configs.All, "MATCH (n) RETURN n.prop").next()("n.prop") shouldBe a[java.lang.Double] + executeWith(Configs.All, "MATCH (n) RETURN n.prop").head("n.prop") shouldBe a[java.lang.Double] } test("should be able to force a type change of a relationship property") { @@ -64,7 +64,7 @@ class SetAcceptanceTest extends ExecutionEngineFunSuite with QueryStatisticsTest // when executeWith(Configs.UpdateConf, "MATCH ()-[r]->() SET r.prop = tofloat(r.prop)") - executeWith(Configs.All, "MATCH ()-[r]->() RETURN r.prop").next()("r.prop") shouldBe a[java.lang.Double] + executeWith(Configs.All, "MATCH ()-[r]->() RETURN r.prop").head("r.prop") shouldBe a[java.lang.Double] } test("should be able to set property to collection") { diff --git a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/ShortestPathAcceptanceTest.scala b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/ShortestPathAcceptanceTest.scala index b4c5db34387f0..90a12a05eeb7f 100644 --- a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/ShortestPathAcceptanceTest.scala +++ b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/ShortestPathAcceptanceTest.scala @@ -648,8 +648,6 @@ class ShortestPathAcceptanceTest extends ExecutionEngineFunSuite with CypherComp result.toSet should equal(Set( Map("n" -> a2.getId, "c" -> 4), Map("n" -> a3.getId, "c" -> 4) )) - - result.close() } test("should work with path expression with 2 repeating bound relationships") { diff --git a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/ShortestPathLongerAcceptanceTest.scala b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/ShortestPathLongerAcceptanceTest.scala index 9720c41b403cc..f12b3accc7fa0 100644 --- a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/ShortestPathLongerAcceptanceTest.scala +++ b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/ShortestPathLongerAcceptanceTest.scala @@ -25,7 +25,7 @@ package org.neo4j.internal.cypher.acceptance import java.util import org.neo4j.cypher.ExecutionEngineFunSuite -import org.neo4j.cypher.internal.runtime.InternalExecutionResult +import org.neo4j.cypher.internal.RewindableExecutionResult import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription import org.neo4j.graphalgo.impl.path.ShortestPath import org.neo4j.graphalgo.impl.path.ShortestPath.DataMonitor @@ -711,7 +711,7 @@ class ShortestPathLongerAcceptanceTest extends ExecutionEngineFunSuite with Cyph dprintln() } - private def evaluateShortestPathResults(results: InternalExecutionResult, startMs: Long, pathLength: Int, expectedNodes: Set[Node]): Unit = { + private def evaluateShortestPathResults(results: RewindableExecutionResult, startMs: Long, pathLength: Int, expectedNodes: Set[Node]): Unit = { val duration = System.currentTimeMillis() - startMs dprintln(results.executionPlanDescription()) @@ -735,7 +735,7 @@ class ShortestPathLongerAcceptanceTest extends ExecutionEngineFunSuite with Cyph private def dprintln() = if (VERBOSE) println private def dprint(s: Any) = if (VERBOSE) print(s) - private def evaluateAllShortestPathResults(results: InternalExecutionResult, identifier: String, startMs: Long, expectedPathCount: Int, expectedNodes: Set[Set[Node]]): Unit = { + private def evaluateAllShortestPathResults(results: RewindableExecutionResult, identifier: String, startMs: Long, expectedPathCount: Int, expectedNodes: Set[Set[Node]]): Unit = { val resultList = results.toList val duration = System.currentTimeMillis() - startMs dprintln(results.executionPlanDescription()) diff --git a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/ShortestPathSameNodeAcceptanceTest.scala b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/ShortestPathSameNodeAcceptanceTest.scala index 1c1299bf1f388..71d59ff0ef1e0 100644 --- a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/ShortestPathSameNodeAcceptanceTest.scala +++ b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/ShortestPathSameNodeAcceptanceTest.scala @@ -119,7 +119,7 @@ class ShortestPathSameNodeAcceptanceTest extends ExecutionEngineFunSuite with Ru } } - def executeUsingCostPlannerOnly(db: GraphDatabaseCypherService, query: String): InternalExecutionResult = { + def executeUsingCostPlannerOnly(db: GraphDatabaseCypherService, query: String): RewindableExecutionResult = { val engine = ExecutionEngineHelper.createEngine(db) RewindableExecutionResult(engine.execute(query, VirtualValues.emptyMap(), diff --git a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/SpatialFunctionsAcceptanceTest.scala b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/SpatialFunctionsAcceptanceTest.scala index 92f6af9920395..1095c9ede6a5e 100644 --- a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/SpatialFunctionsAcceptanceTest.scala +++ b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/SpatialFunctionsAcceptanceTest.scala @@ -98,7 +98,7 @@ class SpatialFunctionsAcceptanceTest extends ExecutionEngineFunSuite with Cypher test("point function should not work with NaN or infinity") { for(invalidDouble <- Seq(Double.NaN, Double.PositiveInfinity, Double.NegativeInfinity)) { - failWithError(Configs.DefaultInterpreted + Configs.SlottedInterpreted + Configs.Version3_4 + Configs.Procs, + failWithError((Configs.Interpreted - Configs.Version2_3) + Configs.Procs, "RETURN point({x: 2.3, y: $v}) as point", List("Cannot create a point with non-finite coordinate values"), params = Map(("v", invalidDouble))) } } diff --git a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/SpatialIndexResultsAcceptanceTest.scala b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/SpatialIndexResultsAcceptanceTest.scala index be24fa4771bd5..50a38cfa25c3c 100644 --- a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/SpatialIndexResultsAcceptanceTest.scala +++ b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/SpatialIndexResultsAcceptanceTest.scala @@ -216,9 +216,10 @@ class SpatialIndexResultsAcceptanceTest extends IndexingTestSupport { graph.execute("CREATE (p:Place) SET p.location = point({latitude: 40.7, longitude: -35.78, crs: 'WGS-84'})") val configuration = TestConfiguration(Versions(Versions.V3_4, Versions.v3_5, Versions.Default), Planners(Planners.Cost, Planners.Default), Runtimes(Runtimes.Interpreted, Runtimes.Slotted, Runtimes.Default)) + val query = "MATCH (p:Place) WHERE p.location = point({latitude: 56.7, longitude: 12.78, crs: 'WGS-84'}) RETURN p.location as point" + // When - val result = executeWith(configuration, - "MATCH (p:Place) WHERE p.location = point({latitude: 56.7, longitude: 12.78, crs: 'WGS-84'}) RETURN p.location as point", + val result = executeWith(configuration + Configs.Cost3_1, query, planComparisonStrategy = ComparePlansWithAssertion({ plan => plan should includeSomewhere .aPlan("Projection").containingArgumentRegex("\\{point : .*\\}".r) diff --git a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/TemporalAcceptanceTest.scala b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/TemporalAcceptanceTest.scala index 8a0ae9375a58c..5f8f5dea95eac 100644 --- a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/TemporalAcceptanceTest.scala +++ b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/TemporalAcceptanceTest.scala @@ -834,10 +834,10 @@ class TemporalAcceptanceTest extends ExecutionEngineFunSuite with QueryStatistic val query = s"RETURN duration('P1Y1M') $op duration('P1Y30D')" withClue(s"Executing $query") { /** - * Version 3.3 returns null instead due to running with 3.4 runtime - * SyntaxException come from the 3.4 planner and IncomparableValuesException from earlier runtimes + * Version 3.3 returns null instead due to running with 3.5 runtime + * SyntaxException come from the 3.5 planner and IncomparableValuesException from earlier runtimes */ - failWithError(Configs.Version3_5 + Configs.Procs - Configs.AllRulePlanners, query, Seq("Type mismatch")) + failWithError(Configs.Version3_5 + Configs.DefaultProcs - Configs.AllRulePlanners, query, Seq("Type mismatch")) } } } diff --git a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/TriadicSelectionAcceptanceTest.scala b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/TriadicSelectionAcceptanceTest.scala index ed015188462b6..a2fe2c10c55e7 100644 --- a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/TriadicSelectionAcceptanceTest.scala +++ b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/TriadicSelectionAcceptanceTest.scala @@ -82,7 +82,7 @@ class TriadicSelectionAcceptanceTest extends ExecutionEngineFunSuite with Cypher |CREATE (a)-[:FRIEND]->(b), (b)-[:FRIEND]->(c), (c)-[:FRIEND]->(a)""".stripMargin) // when - val result: InternalExecutionResult = executeWith(configs, QUERY, planComparisonStrategy = usesTriadic) + val result = executeWith(configs, QUERY, planComparisonStrategy = usesTriadic) // then result should be(empty) diff --git a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/UnsupportedFeaturesAcceptanceTest.scala b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/UnsupportedFeaturesAcceptanceTest.scala index 84bbeef5c84ee..e89657472bf6c 100644 --- a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/UnsupportedFeaturesAcceptanceTest.scala +++ b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/UnsupportedFeaturesAcceptanceTest.scala @@ -28,7 +28,7 @@ import org.neo4j.internal.cypher.acceptance.CypherComparisonSupport._ import scala.language.postfixOps class UnsupportedFeaturesAcceptanceTest extends ExecutionEngineFunSuite with CypherComparisonSupport { - val configs = Configs.Version3_5 + Configs.Procs - Configs.AllRulePlanners + val configs = Configs.Version3_5 + Configs.DefaultProcs - Configs.AllRulePlanners test("from graph") { val query = "FROM GRAPH foo.bar MATCH (a)-->() RETURN a" @@ -59,4 +59,4 @@ class UnsupportedFeaturesAcceptanceTest extends ExecutionEngineFunSuite with Cyp val query = "RETURN 1 ~ 2" failWithError(configs, query, List("`~` (equivalence) is a Cypher 10 feature and is not available in this implementation of Cypher.")) } -} \ No newline at end of file +} diff --git a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/UpdateReportingAcceptanceTest.scala b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/UpdateReportingAcceptanceTest.scala index c8d0ba49be29c..06c61e3a7a090 100644 --- a/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/UpdateReportingAcceptanceTest.scala +++ b/enterprise/cypher/acceptance-spec-suite/src/test/scala/org/neo4j/internal/cypher/acceptance/UpdateReportingAcceptanceTest.scala @@ -27,7 +27,7 @@ import org.neo4j.internal.cypher.acceptance.CypherComparisonSupport.Configs class UpdateReportingAcceptanceTest extends ExecutionEngineFunSuite with CypherComparisonSupport { test("creating a node gets reported as such") { - val output = executeWith(Configs.UpdateConf, "create (:A)").dumpToString() + val output = dumpToString(Configs.UpdateConf + Configs.DefaultProcs, "create (:A)") output should include("Nodes created: 1") output should include("Labels added: 1") diff --git a/enterprise/cypher/cypher/src/main/java/org/neo4j/cypher/internal/codegen/profiling/ProfilingTracer.java b/enterprise/cypher/cypher/src/main/java/org/neo4j/cypher/internal/codegen/profiling/ProfilingTracer.java index 93686148be205..7b87631f9e2ae 100644 --- a/enterprise/cypher/cypher/src/main/java/org/neo4j/cypher/internal/codegen/profiling/ProfilingTracer.java +++ b/enterprise/cypher/cypher/src/main/java/org/neo4j/cypher/internal/codegen/profiling/ProfilingTracer.java @@ -22,30 +22,19 @@ */ package org.neo4j.cypher.internal.codegen.profiling; +import org.opencypher.v9_0.util.attribution.Id; + import java.util.HashMap; import java.util.Map; -import org.neo4j.cypher.internal.runtime.compiled.codegen.QueryExecutionEvent; -import org.neo4j.cypher.internal.planner.v3_5.spi.KernelStatisticProvider; -import org.opencypher.v9_0.util.attribution.Id; import org.neo4j.cypher.internal.codegen.QueryExecutionTracer; -import org.neo4j.helpers.MathUtil; +import org.neo4j.cypher.internal.planner.v3_5.spi.KernelStatisticProvider; +import org.neo4j.cypher.internal.runtime.compiled.codegen.QueryExecutionEvent; +import org.neo4j.cypher.result.OperatorProfile; +import org.neo4j.cypher.result.QueryProfile; -public class ProfilingTracer implements QueryExecutionTracer +public class ProfilingTracer implements QueryExecutionTracer, QueryProfile { - public interface ProfilingInformation - { - long time(); - long dbHits(); - long rows(); - long pageCacheHits(); - long pageCacheMisses(); - default double pageCacheHitRatio() - { - return MathUtil.portion( pageCacheHits(), pageCacheMisses() ); - } - } - public interface Clock { long nanoTime(); @@ -57,7 +46,7 @@ public interface Clock private final Clock clock; private final KernelStatisticProvider statisticProvider; - private final Map data = new HashMap<>(); + private final Map data = new HashMap<>(); public ProfilingTracer( KernelStatisticProvider statisticProvider ) { @@ -70,7 +59,7 @@ public ProfilingTracer( KernelStatisticProvider statisticProvider ) this.statisticProvider = statisticProvider; } - public ProfilingInformation get( Id query ) + public OperatorProfile operatorProfile( int query ) { Data value = data.get( query ); return value == null ? ZERO : value; @@ -78,27 +67,27 @@ public ProfilingInformation get( Id query ) public long timeOf( Id query ) { - return get( query ).time(); + return operatorProfile( query.x() ).time(); } public long dbHitsOf( Id query ) { - return get( query ).dbHits(); + return operatorProfile( query.x() ).dbHits(); } public long rowsOf( Id query ) { - return get( query ).rows(); + return operatorProfile( query.x() ).rows(); } @Override public QueryExecutionEvent executeOperator( Id queryId ) { - Data queryData = this.data.get( queryId ); - if ( queryData == null && queryId != null ) + Data queryData = this.data.get( queryId.x() ); + if ( queryData == null ) { queryData = new Data(); - this.data.put( queryId, queryData ); + this.data.put( queryId.x(), queryData ); } return new ExecutionEvent( clock, statisticProvider, queryData ); } @@ -145,7 +134,7 @@ public void row() } } - private static class Data implements ProfilingInformation + private static class Data implements OperatorProfile { private long time; private long hits; diff --git a/enterprise/cypher/cypher/src/main/java/org/neo4j/cypher/internal/executionplan/GeneratedQueryExecution.java b/enterprise/cypher/cypher/src/main/java/org/neo4j/cypher/internal/executionplan/GeneratedQueryExecution.java index eda32c4da522e..15f531e364055 100644 --- a/enterprise/cypher/cypher/src/main/java/org/neo4j/cypher/internal/executionplan/GeneratedQueryExecution.java +++ b/enterprise/cypher/cypher/src/main/java/org/neo4j/cypher/internal/executionplan/GeneratedQueryExecution.java @@ -23,7 +23,6 @@ package org.neo4j.cypher.internal.executionplan; import org.neo4j.cypher.internal.runtime.ExecutionMode; -import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.Completable; import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription; import org.neo4j.cypher.result.QueryResult; @@ -36,6 +35,4 @@ public interface GeneratedQueryExecution InternalPlanDescription executionPlanDescription(); String[] fieldNames(); - - void setCompletable( Completable completable ); } diff --git a/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/CompiledRuntime.scala b/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/CompiledRuntime.scala index 9286d5ad53e31..b6b67215f8672 100644 --- a/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/CompiledRuntime.scala +++ b/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/CompiledRuntime.scala @@ -24,22 +24,17 @@ package org.neo4j.cypher.internal import org.neo4j.cypher.internal.codegen.profiling.ProfilingTracer import org.neo4j.cypher.internal.compatibility.CypherRuntime -import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.{Provider, ExecutionPlan => ExecutionPlanv3_5} +import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.{ExecutionPlan => ExecutionPlanv3_5} import org.neo4j.cypher.internal.compatibility.v3_5.runtime.helpers.InternalWrapping.asKernelNotification -import org.neo4j.cypher.internal.compatibility.v3_5.runtime.{CompiledRuntimeName, ExplainExecutionResult, RuntimeName} +import org.neo4j.cypher.internal.compatibility.v3_5.runtime.{CompiledRuntimeName, RuntimeName} import org.neo4j.cypher.internal.compiler.v3_5.phases.LogicalPlanState import org.neo4j.cypher.internal.compiler.v3_5.planner.CantCompileQueryException import org.neo4j.cypher.internal.runtime._ -import org.neo4j.cypher.internal.runtime.compiled.ExecutionPlanBuilder.DescriptionProvider -import org.neo4j.cypher.internal.runtime.compiled.codegen.{CodeGenConfiguration, CodeGenerator} import org.neo4j.cypher.internal.runtime.compiled.CompiledPlan -import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription -import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription.Arguments -import org.neo4j.cypher.internal.v3_5.logical.plans.IndexUsage +import org.neo4j.cypher.internal.runtime.compiled.codegen.{CodeGenConfiguration, CodeGenerator} +import org.neo4j.cypher.result.RuntimeResult import org.neo4j.graphdb.Notification import org.neo4j.values.virtual.MapValue -import org.opencypher.v9_0.frontend.PlannerName -import org.opencypher.v9_0.util.TaskCloser object CompiledRuntime extends CypherRuntime[EnterpriseRuntimeContext] { @@ -70,44 +65,17 @@ object CompiledRuntime extends CypherRuntime[EnterpriseRuntimeContext] { val notifications: Set[Notification]) extends ExecutionPlanv3_5 { override def run(queryContext: QueryContext, - executionMode: ExecutionMode, params: MapValue): InternalExecutionResult = { - val taskCloser = new TaskCloser - taskCloser.addTask(queryContext.transactionalContext.close) - try { - if (executionMode == ExplainMode) { - //close all statements - taskCloser.close(success = true) - ExplainExecutionResult(compiled.columns.toArray, compiled.planDescription.get(), READ_ONLY, notifications) - } else - compiled.executionResultBuilder(queryContext, executionMode, createTracer(executionMode, queryContext), params, taskCloser) - } catch { - case t: Throwable => - taskCloser.close(success = false) - throw t - } - } + doProfile: Boolean, + params: MapValue): RuntimeResult = { - override val runtimeName: RuntimeName = CompiledRuntimeName - } + val executionMode = if (doProfile) ProfileMode else NormalMode + val tracer = + if (doProfile) Some(new ProfilingTracer(queryContext.transactionalContext.kernelStatisticProvider)) + else None - private def createTracer(mode: ExecutionMode, queryContext: QueryContext): DescriptionProvider = mode match { - case ProfileMode => - val tracer = new ProfilingTracer(queryContext.transactionalContext.kernelStatisticProvider) - (description: Provider[InternalPlanDescription]) => - (new Provider[InternalPlanDescription] { + compiled.executionResultBuilder(queryContext, executionMode, tracer, params) + } - override def get(): InternalPlanDescription = description.get().map { - plan: InternalPlanDescription => - val data = tracer.get(plan.id) - plan. - addArgument(Arguments.DbHits(data.dbHits())). - addArgument(Arguments.PageCacheHits(data.pageCacheHits())). - addArgument(Arguments.PageCacheMisses(data.pageCacheMisses())). - addArgument(Arguments.PageCacheHitRatio(data.pageCacheHitRatio())). - addArgument(Arguments.Rows(data.rows())). - addArgument(Arguments.Time(data.time())) - } - }, Some(tracer)) - case _ => (description: Provider[InternalPlanDescription]) => (description, None) + override val runtimeName: RuntimeName = CompiledRuntimeName } } diff --git a/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/MorselRuntime.scala b/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/MorselRuntime.scala index 012f28a0ef7d8..4ffd72c4b429a 100644 --- a/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/MorselRuntime.scala +++ b/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/MorselRuntime.scala @@ -26,28 +26,21 @@ import org.neo4j.cypher.internal.compatibility.CypherRuntime import org.neo4j.cypher.internal.compatibility.v3_5.runtime.PhysicalPlanningAttributes.SlotConfigurations import org.neo4j.cypher.internal.compatibility.v3_5.runtime.SlotAllocation.PhysicalPlan import org.neo4j.cypher.internal.compatibility.v3_5.runtime._ -import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.StandardInternalExecutionResult.IterateByAccepting -import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.{StandardInternalExecutionResult, ExecutionPlan => ExecutionPlan_V35} -import org.neo4j.cypher.internal.compatibility.v3_5.runtime.helpers.InternalWrapping.asKernelNotification +import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.{ExecutionPlan => ExecutionPlan_V35} import org.neo4j.cypher.internal.compiler.v3_5.ExperimentalFeatureNotification import org.neo4j.cypher.internal.compiler.v3_5.phases.LogicalPlanState -import org.neo4j.cypher.internal.planner.v3_5.spi.PlanningAttributes.Cardinalities import org.neo4j.cypher.internal.runtime._ import org.neo4j.cypher.internal.runtime.interpreted.commands.convert.{CommunityExpressionConverter, ExpressionConverters} -import org.neo4j.cypher.internal.runtime.parallel._ -import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription.Arguments.{Runtime, RuntimeImpl} -import org.neo4j.cypher.internal.runtime.planDescription.{InternalPlanDescription, LogicalPlan2PlanDescription} +import org.neo4j.cypher.internal.runtime.parallel.SchedulerTracer import org.neo4j.cypher.internal.runtime.slotted.expressions.{CompiledExpressionConverter, SlottedExpressionConverters} import org.neo4j.cypher.internal.runtime.vectorized.expressions.MorselExpressionConverters import org.neo4j.cypher.internal.runtime.vectorized.{Dispatcher, Pipeline, PipelineBuilder} import org.neo4j.cypher.internal.v3_5.logical.plans.LogicalPlan import org.neo4j.cypher.result.QueryResult.QueryResultVisitor -import org.neo4j.graphdb.Notification +import org.neo4j.cypher.result.{QueryProfile, RuntimeResult} +import org.neo4j.graphdb.ResourceIterator import org.neo4j.values.virtual.MapValue import org.opencypher.v9_0.ast.semantics.SemanticTable -import org.opencypher.v9_0.frontend.PlannerName -import org.opencypher.v9_0.frontend.phases.InternalNotificationLogger -import org.opencypher.v9_0.util.TaskCloser object MorselRuntime extends CypherRuntime[EnterpriseRuntimeContext] { override def compileToExecutable(state: LogicalPlanState, context: EnterpriseRuntimeContext): ExecutionPlan_V35 = { @@ -68,16 +61,12 @@ object MorselRuntime extends CypherRuntime[EnterpriseRuntimeContext] { ExperimentalFeatureNotification("use the morsel runtime at your own peril, " + "not recommended to be run on production systems")) - VectorizedExecutionPlan(state.plannerName, - operators, - physicalPlan.slotConfigurations, - logicalPlan, - fieldNames, - dispatcher, - context.notificationLogger, - context.readOnly, - state.cardinalities, - tracer) + VectorizedExecutionPlan(operators, + physicalPlan.slotConfigurations, + logicalPlan, + fieldNames, + dispatcher, + tracer) } private def rewritePlan(context: EnterpriseRuntimeContext, beforeRewrite: LogicalPlan, semanticTable: SemanticTable) = { @@ -87,68 +76,55 @@ object MorselRuntime extends CypherRuntime[EnterpriseRuntimeContext] { (logicalPlan, physicalPlan) } - case class VectorizedExecutionPlan(plannerUsed: PlannerName, - operators: Pipeline, + case class VectorizedExecutionPlan(operators: Pipeline, slots: SlotConfigurations, logicalPlan: LogicalPlan, fieldNames: Array[String], dispatcher: Dispatcher, - notificationLogger: InternalNotificationLogger, - readOnly: Boolean, - cardinalities: Cardinalities, schedulerTracer: SchedulerTracer) extends ExecutionPlan_V35 { - override def run(queryContext: QueryContext, planType: ExecutionMode, params: MapValue): InternalExecutionResult = { - val taskCloser = new TaskCloser - taskCloser.addTask(queryContext.transactionalContext.close) - taskCloser.addTask(queryContext.resources.close) - val planDescription = - () => LogicalPlan2PlanDescription(logicalPlan, plannerUsed, readOnly, cardinalities) - .addArgument(Runtime(MorselRuntimeName.toTextOutput)) - .addArgument(RuntimeImpl(MorselRuntimeName.name)) - - if (planType == ExplainMode) { - //close all statements - taskCloser.close(success = true) - ExplainExecutionResult(fieldNames, planDescription(), READ_ONLY, - notificationLogger.notifications.map(asKernelNotification(notificationLogger.offset))) - } else new VectorizedOperatorExecutionResult(operators, - logicalPlan, - planDescription, - queryContext, - params, - fieldNames, - taskCloser, - dispatcher, - schedulerTracer) + + override def run(queryContext: QueryContext, + doProfile: Boolean, + params: MapValue): RuntimeResult = { + + new VectorizedRuntimeResult(operators, + logicalPlan, + queryContext, + params, + fieldNames, + dispatcher, + schedulerTracer) } override def runtimeName: RuntimeName = MorselRuntimeName } - class VectorizedOperatorExecutionResult(operators: Pipeline, - logicalPlan: LogicalPlan, - executionPlanBuilder: () => InternalPlanDescription, - queryContext: QueryContext, - params: MapValue, - override val fieldNames: Array[String], - taskCloser: TaskCloser, - dispatcher: Dispatcher, - schedulerTracer: SchedulerTracer) extends StandardInternalExecutionResult(queryContext, ProcedureRuntimeName, Some(taskCloser)) with IterateByAccepting { + class VectorizedRuntimeResult(operators: Pipeline, + logicalPlan: LogicalPlan, + queryContext: QueryContext, + params: MapValue, + override val fieldNames: Array[String], + dispatcher: Dispatcher, + schedulerTracer: SchedulerTracer) extends RuntimeResult { + private var isDone = false - override def accept[E <: Exception](visitor: QueryResultVisitor[E]): Unit = - dispatcher.execute(operators, queryContext, params, taskCloser, schedulerTracer)(visitor) + override def accept[E <: Exception](visitor: QueryResultVisitor[E]): Unit = { + dispatcher.execute(operators, queryContext, params, schedulerTracer)(visitor) + isDone = true + } override def queryStatistics(): runtime.QueryStatistics = queryContext.getOptStatistics.getOrElse(QueryStatistics()) - override def executionPlanDescription(): InternalPlanDescription = executionPlanBuilder() + override def isIterable: Boolean = false - override def queryType: InternalQueryType = READ_ONLY + override def asIterator(): ResourceIterator[java.util.Map[String, AnyRef]] = + throw new UnsupportedOperationException("The Morsel runtime is not iterable") - override def executionMode: ExecutionMode = NormalMode + override def isExhausted: Boolean = isDone - override def notifications: Iterable[Notification] = Iterable.empty[Notification] + override def close(): Unit = {} - override def withNotifications(notification: Notification*): InternalExecutionResult = this + override def queryProfile(): QueryProfile = QueryProfile.NONE } } diff --git a/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/SlottedRuntime.scala b/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/SlottedRuntime.scala index 991342abc936d..1cc301cd431a2 100644 --- a/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/SlottedRuntime.scala +++ b/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/SlottedRuntime.scala @@ -79,7 +79,7 @@ object SlottedRuntime extends CypherRuntime[EnterpriseRuntimeContext] with Debug val pipeBuilderFactory = SlottedPipeBuilder.Factory(physicalPlan) val executionPlanBuilder = new PipeExecutionPlanBuilder(expressionConverters = converters, pipeBuilderFactory = pipeBuilderFactory) - val pipeBuildContext = PipeExecutionBuilderContext(state.semanticTable(), context.readOnly, state.cardinalities) + val pipeBuildContext = PipeExecutionBuilderContext(state.semanticTable(), context.readOnly) val pipe = executionPlanBuilder.build(logicalPlan)(pipeBuildContext, context.tokenContext) val periodicCommitInfo = state.periodicCommit.map(x => PeriodicCommitInfo(x.batchSize)) val columns = state.statement().returnColumns @@ -99,10 +99,8 @@ object SlottedRuntime extends CypherRuntime[EnterpriseRuntimeContext] with Debug periodicCommitInfo, resultBuilderFactory, context.notificationLogger, - state.plannerName, SlottedRuntimeName, - context.readOnly, - state.cardinalities) + context.readOnly) } catch { case e: CypherException => diff --git a/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/runtime/compiled/CompiledExecutionResult.scala b/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/runtime/compiled/CompiledExecutionResult.scala index 1839b03345e6b..9a8d4dad97157 100644 --- a/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/runtime/compiled/CompiledExecutionResult.scala +++ b/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/runtime/compiled/CompiledExecutionResult.scala @@ -22,54 +22,41 @@ */ package org.neo4j.cypher.internal.runtime.compiled -import org.neo4j.cypher.internal.compatibility.v3_5.runtime.CompiledRuntimeName -import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.{Provider, StandardInternalExecutionResult} +import java.util + import org.neo4j.cypher.internal.executionplan.GeneratedQueryExecution import org.neo4j.cypher.internal.runtime._ -import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription -import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription.Arguments.{Runtime, RuntimeImpl} -import org.opencypher.v9_0.util.{ProfilerStatisticsNotReadyException, TaskCloser} import org.neo4j.cypher.result.QueryResult.QueryResultVisitor -import org.neo4j.graphdb.Notification +import org.neo4j.cypher.result.{QueryProfile, RuntimeResult} +import org.neo4j.graphdb.ResourceIterator /** - * Main class for compiled execution results, implements everything in InternalExecutionResult - * except `javaColumns` and `accept` which delegates to the injected compiled code. + * Main class for compiled runtime results. */ -class CompiledExecutionResult(taskCloser: TaskCloser, - context: QueryContext, +class CompiledExecutionResult(context: QueryContext, compiledCode: GeneratedQueryExecution, - description: Provider[InternalPlanDescription], - notifications: Iterable[Notification] = Iterable.empty) - extends StandardInternalExecutionResult(context, CompiledRuntimeName, Some(taskCloser)) - with StandardInternalExecutionResult.IterateByAccepting { + override val queryProfile: QueryProfile) + extends RuntimeResult { - compiledCode.setCompletable(this) + private var isDone = false - // *** Delegate to compiled code def executionMode: ExecutionMode = compiledCode.executionMode() override def fieldNames(): Array[String] = compiledCode.fieldNames() - override def accept[EX <: Exception](visitor: QueryResultVisitor[EX]): Unit = + override def accept[EX <: Exception](visitor: QueryResultVisitor[EX]): Unit = { compiledCode.accept(visitor) - - override def executionPlanDescription(): InternalPlanDescription = { - if (!taskCloser.isClosed && executionMode == ProfileMode) { - completed(success = false) - throw new ProfilerStatisticsNotReadyException - } - - compiledCode.executionPlanDescription() - .addArgument(Runtime(CompiledRuntimeName.toTextOutput)) - .addArgument(RuntimeImpl(CompiledRuntimeName.name)) + isDone = true } override def queryStatistics() = QueryStatistics() - //TODO delegate to compiled code once writes are being implemented - override def queryType: InternalQueryType = READ_ONLY + override def isIterable: Boolean = false + + override def asIterator(): ResourceIterator[util.Map[String, AnyRef]] = + throw new UnsupportedOperationException("The compiled runtime is not iterable") + + override def isExhausted: Boolean = isDone - override def withNotifications(notification: Notification*): InternalExecutionResult = - new CompiledExecutionResult(taskCloser, context, compiledCode, description, notification) + override def close(): Unit = {} } diff --git a/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/runtime/compiled/ExecutionPlanBuilder.scala b/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/runtime/compiled/ExecutionPlanBuilder.scala deleted file mode 100644 index 1e5ab6c9f9459..0000000000000 --- a/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/runtime/compiled/ExecutionPlanBuilder.scala +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2002-2018 "Neo4j," - * Neo4j Sweden AB [http://neo4j.com] - * - * This file is part of Neo4j Enterprise Edition. The included source - * code can be redistributed and/or modified under the terms of the - * GNU AFFERO GENERAL PUBLIC LICENSE Version 3 - * (http://www.fsf.org/licensing/licenses/agpl-3.0.html) with the - * Commons Clause, as found in the associated LICENSE.txt file. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * Neo4j object code can be licensed independently from the source - * under separate terms from the AGPL. Inquiries can be directed to: - * licensing@neo4j.com - * - * More information is also available at: - * https://neo4j.com/licensing/ - */ -package org.neo4j.cypher.internal.runtime.compiled - -import org.neo4j.cypher.internal.codegen.QueryExecutionTracer -import org.neo4j.cypher.internal.codegen.profiling.ProfilingTracer -import org.neo4j.cypher.internal.compatibility.v3_5.runtime.CompiledRuntimeName -import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.Provider -import org.neo4j.cypher.internal.runtime.compiled.ExecutionPlanBuilder.DescriptionProvider -import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription -import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription.Arguments -import org.neo4j.cypher.internal.runtime.{ExecutionMode, InternalExecutionResult, ProfileMode, QueryContext} -import org.neo4j.values.virtual.MapValue -import org.opencypher.v9_0.util.TaskCloser - -object ExecutionPlanBuilder { - type DescriptionProvider = - (Provider[InternalPlanDescription] => (Provider[InternalPlanDescription], Option[QueryExecutionTracer])) - - def tracer(mode: ExecutionMode, queryContext: QueryContext): DescriptionProvider = mode match { - case ProfileMode => - val tracer = new ProfilingTracer(queryContext.transactionalContext.kernelStatisticProvider) - (description: Provider[InternalPlanDescription]) => - (new Provider[InternalPlanDescription] { - - override def get(): InternalPlanDescription = description.get().map { - plan: InternalPlanDescription => - val data = tracer.get(plan.id) - - plan. - addArgument(Arguments.Runtime(CompiledRuntimeName.toTextOutput)). - addArgument(Arguments.DbHits(data.dbHits())). - addArgument(Arguments.PageCacheHits(data.pageCacheHits())). - addArgument(Arguments.PageCacheMisses(data.pageCacheMisses())). - addArgument(Arguments.Rows(data.rows())). - addArgument(Arguments.Time(data.time())) - } - }, Some(tracer)) - case _ => (description: Provider[InternalPlanDescription]) => (description, None) - } -} - -case class CompiledPlan(updating: Boolean, - planDescription: Provider[InternalPlanDescription], - columns: Seq[String], - executionResultBuilder: RunnablePlan) - -trait RunnablePlan { - def apply(queryContext: QueryContext, - execMode: ExecutionMode, - descriptionProvider: DescriptionProvider, - params: MapValue, - closer: TaskCloser): InternalExecutionResult -} diff --git a/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/runtime/compiled/RunnablePlan.scala b/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/runtime/compiled/RunnablePlan.scala new file mode 100644 index 0000000000000..1e6ca5bb5bece --- /dev/null +++ b/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/runtime/compiled/RunnablePlan.scala @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2002-2018 "Neo4j," + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j Enterprise Edition. The included source + * code can be redistributed and/or modified under the terms of the + * GNU AFFERO GENERAL PUBLIC LICENSE Version 3 + * (http://www.fsf.org/licensing/licenses/agpl-3.0.html) with the + * Commons Clause, as found in the associated LICENSE.txt file. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * Neo4j object code can be licensed independently from the source + * under separate terms from the AGPL. Inquiries can be directed to: + * licensing@neo4j.com + * + * More information is also available at: + * https://neo4j.com/licensing/ + */ +package org.neo4j.cypher.internal.runtime.compiled + +import org.neo4j.cypher.internal.codegen.profiling.ProfilingTracer +import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.Provider +import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription +import org.neo4j.cypher.internal.runtime.{ExecutionMode, QueryContext} +import org.neo4j.cypher.result.RuntimeResult +import org.neo4j.values.virtual.MapValue + +case class CompiledPlan(updating: Boolean, + planDescription: Provider[InternalPlanDescription], + columns: Seq[String], + executionResultBuilder: RunnablePlan) + +trait RunnablePlan { + def apply(queryContext: QueryContext, + execMode: ExecutionMode, + tracer: Option[ProfilingTracer], + params: MapValue): RuntimeResult +} diff --git a/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/runtime/compiled/codegen/CodeGenerator.scala b/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/runtime/compiled/codegen/CodeGenerator.scala index 5b7a01dbafe82..8f2a8ada032f3 100644 --- a/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/runtime/compiled/codegen/CodeGenerator.scala +++ b/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/runtime/compiled/codegen/CodeGenerator.scala @@ -26,25 +26,26 @@ import java.time.Clock import java.util import org.neo4j.cypher.internal.codegen.QueryExecutionTracer +import org.neo4j.cypher.internal.codegen.profiling.ProfilingTracer import org.neo4j.cypher.internal.compatibility.v3_5.runtime.CompiledRuntimeName import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.Provider import org.neo4j.cypher.internal.compiler.v3_5.planner.CantCompileQueryException import org.neo4j.cypher.internal.executionplan.{GeneratedQuery, GeneratedQueryExecution} import org.neo4j.cypher.internal.planner.v3_5.spi.PlanningAttributes.Cardinalities import org.neo4j.cypher.internal.planner.v3_5.spi.TokenContext -import org.neo4j.cypher.internal.runtime.compiled.ExecutionPlanBuilder.DescriptionProvider import org.neo4j.cypher.internal.runtime.compiled.codegen.ir._ import org.neo4j.cypher.internal.runtime.compiled.codegen.spi.{CodeStructure, CodeStructureResult} import org.neo4j.cypher.internal.runtime.compiled.{CompiledExecutionResult, CompiledPlan, RunnablePlan} import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription.Arguments.{Runtime, RuntimeImpl} import org.neo4j.cypher.internal.runtime.planDescription.{InternalPlanDescription, LogicalPlan2PlanDescription} -import org.neo4j.cypher.internal.runtime.{ExecutionMode, InternalExecutionResult, QueryContext, compiled} +import org.neo4j.cypher.internal.runtime.{ExecutionMode, QueryContext, compiled} import org.neo4j.cypher.internal.v3_5.logical.plans.{LogicalPlan, ProduceResult} +import org.neo4j.cypher.result.{QueryProfile, RuntimeResult} import org.neo4j.values.virtual.MapValue import org.opencypher.v9_0.ast.semantics.SemanticTable import org.opencypher.v9_0.frontend.PlannerName +import org.opencypher.v9_0.util.Eagerly import org.opencypher.v9_0.util.attribution.Id -import org.opencypher.v9_0.util.{Eagerly, TaskCloser} class CodeGenerator(val structure: CodeStructure[GeneratedQuery], clock: Clock, @@ -82,14 +83,18 @@ class CodeGenerator(val structure: CodeStructure[GeneratedQuery], } val builder = new RunnablePlan { - def apply(queryContext: QueryContext, execMode: ExecutionMode, - descriptionProvider: DescriptionProvider, params: MapValue, - closer: TaskCloser): InternalExecutionResult = { - val (provider, tracer) = descriptionProvider(description) - val execution: GeneratedQueryExecution = query.query.execute(queryContext, execMode, provider, + def apply(queryContext: QueryContext, + execMode: ExecutionMode, + tracer: Option[ProfilingTracer], + params: MapValue): RuntimeResult = { + val explodingProvider = + new Provider[InternalPlanDescription] { + override def get(): InternalPlanDescription = ??? + } + + val execution: GeneratedQueryExecution = query.query.execute(queryContext, execMode, explodingProvider, tracer.getOrElse(QueryExecutionTracer.NONE),params) - closer.addTask(queryContext.resources.close) - new CompiledExecutionResult(closer, queryContext, execution, provider) + new CompiledExecutionResult(queryContext, execution, tracer.getOrElse(QueryProfile.NONE)) } } diff --git a/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/spi/codegen/Fields.scala b/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/spi/codegen/Fields.scala index cd7484faf639d..28f82cd0c30af 100644 --- a/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/spi/codegen/Fields.scala +++ b/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/spi/codegen/Fields.scala @@ -29,7 +29,6 @@ case class Fields(entityAccessor: FieldReference, description: FieldReference, tracer: FieldReference, params: FieldReference, - closeable: FieldReference, queryContext: FieldReference, skip: FieldReference, cursors: FieldReference, diff --git a/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/spi/codegen/GeneratedQueryStructure.scala b/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/spi/codegen/GeneratedQueryStructure.scala index 916da77f6c212..fad2c319ea583 100644 --- a/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/spi/codegen/GeneratedQueryStructure.scala +++ b/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/spi/codegen/GeneratedQueryStructure.scala @@ -33,7 +33,7 @@ import org.neo4j.codegen.source.SourceCode.SOURCECODE import org.neo4j.codegen.source.{SourceCode, SourceVisitor} import org.neo4j.codegen.{CodeGenerator, Parameter, TypeReference, _} import org.neo4j.cypher.internal.codegen.{PrimitiveNodeStream, PrimitiveRelationshipStream, QueryExecutionTracer} -import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.{Completable, Provider} +import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.Provider import org.neo4j.cypher.internal.executionplan.{GeneratedQuery, GeneratedQueryExecution} import org.opencypher.v9_0.frontend.helpers.using import org.neo4j.cypher.internal.javacompat.ResultRecord @@ -161,9 +161,6 @@ object GeneratedQueryStructure extends CodeStructure[GeneratedQuery] { val structure = new GeneratedMethodStructure(fields, codeBlock, new AuxGenerator(conf.packageName, generator), onClose = Seq((success: Boolean) => (block: CodeBlock) => { block.expression(invoke(block.self(), methodReference(block.owner(), TypeReference.VOID, "closeCursors"))) - val target = Expression.get(block.self(), fields.closeable) - val reference = method[Completable, Unit]("completed", typeRef[Boolean]) - block.expression(invoke(target, reference, Expression.constant(success))) })) codeBlock.assign(typeRef[ResultRecord], "row", invoke(newInstance(typeRef[ResultRecord]), @@ -185,7 +182,6 @@ object GeneratedQueryStructure extends CodeStructure[GeneratedQuery] { Templates.relationshipScanCursor(clazz, fields) Templates.propertyCursor(clazz, fields) Templates.closeCursors(clazz, fields) - clazz.generate(Templates.setCompletable(clazz.handle())) clazz.generate(Templates.executionMode(clazz.handle())) clazz.generate(Templates.executionPlanDescription(clazz.handle())) clazz.generate(Templates.FIELD_NAMES) @@ -207,7 +203,6 @@ object GeneratedQueryStructure extends CodeStructure[GeneratedQuery] { description = clazz.field(typeRef[Provider[InternalPlanDescription]], "description"), tracer = clazz.field(typeRef[QueryExecutionTracer], "tracer"), params = clazz.field(typeRef[MapValue], "params"), - closeable = clazz.field(typeRef[Completable], "closeable"), queryContext = clazz.field(typeRef[QueryContext], "queryContext"), skip = clazz.field(typeRef[Boolean], "skip"), cursors = clazz.field(typeRef[CursorFactory], "cursors"), diff --git a/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/spi/codegen/Templates.scala b/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/spi/codegen/Templates.scala index 4a301eb951f30..1f973c97531f9 100644 --- a/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/spi/codegen/Templates.scala +++ b/enterprise/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/spi/codegen/Templates.scala @@ -33,7 +33,7 @@ import org.neo4j.codegen.MethodDeclaration.Builder import org.neo4j.codegen.MethodReference._ import org.neo4j.codegen._ import org.neo4j.cypher.internal.codegen.{PrimitiveNodeStream, PrimitiveRelationshipStream, QueryExecutionTracer} -import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.{Completable, Provider} +import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.Provider import org.opencypher.v9_0.frontend.helpers.using import org.neo4j.cypher.internal.javacompat.ResultRowImpl import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription @@ -284,11 +284,6 @@ object Templates { } } - def setCompletable(classHandle: ClassHandle) = MethodTemplate.method(typeRef[Unit], "setCompletable", - param[Completable]("closeable")). - put(self(classHandle), typeRef[Completable], "closeable", load("closeable", typeRef[Completable])). - build() - def executionMode(classHandle: ClassHandle) = MethodTemplate.method(typeRef[ExecutionMode], "executionMode"). returns(get(self(classHandle), typeRef[ExecutionMode], "executionMode")). build() diff --git a/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/RootPlanAcceptanceTest.scala b/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/RootPlanAcceptanceTest.scala index 244c9d6ae3c7d..3bd17069d1dcd 100644 --- a/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/RootPlanAcceptanceTest.scala +++ b/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/RootPlanAcceptanceTest.scala @@ -23,11 +23,9 @@ package org.neo4j.cypher import org.neo4j.cypher.internal.compatibility.v3_5.runtime.{CompiledRuntimeName, InterpretedRuntimeName, RuntimeName, SlottedRuntimeName} -import org.neo4j.cypher.internal.compiler.v3_5._ -import org.opencypher.v9_0.frontend.PlannerName import org.neo4j.cypher.internal.planner.v3_5.spi.{CostBasedPlannerName, DPPlannerName, IDPPlannerName} import org.neo4j.graphdb.ExecutionPlanDescription -import org.neo4j.values.virtual.VirtualValues +import org.opencypher.v9_0.frontend.PlannerName class RootPlanAcceptanceTest extends ExecutionEngineFunSuite { @@ -164,9 +162,9 @@ class RootPlanAcceptanceTest extends ExecutionEngineFunSuite { val runtimeString = runtime.map("runtime=" + _.name).getOrElse("") s"CYPHER $version $plannerString $runtimeString" } - val result = profile(s"$prepend $query") - result.dumpToString() - result.executionPlanDescription() + val result = executeOfficial(s"$prepend PROFILE $query") + result.resultAsString() + result.getExecutionPlanDescription() } } } diff --git a/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/codegen/profiling/ProfilingTracerTest.scala b/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/codegen/profiling/ProfilingTracerTest.scala index 82da6fe9f38e9..9ad01d11e03aa 100644 --- a/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/codegen/profiling/ProfilingTracerTest.scala +++ b/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/codegen/profiling/ProfilingTracerTest.scala @@ -124,7 +124,7 @@ class ProfilingTracerTest extends CypherFunSuite { event.close() - val information = tracer.get(operatorId) + val information = tracer.operatorProfile(operatorId.x) information.pageCacheHits() should equal(100) } @@ -144,7 +144,7 @@ class ProfilingTracerTest extends CypherFunSuite { event.close() - val information = tracer.get(operatorId) + val information = tracer.operatorProfile(operatorId.x) information.pageCacheMisses() should equal(17) } diff --git a/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/compiled_runtime/TypeConversionTest.scala b/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/compiled_runtime/TypeConversionTest.scala index 9538e17c2be3a..2cb45b8ec86ac 100644 --- a/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/compiled_runtime/TypeConversionTest.scala +++ b/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/compiled_runtime/TypeConversionTest.scala @@ -22,7 +22,7 @@ */ package org.neo4j.cypher.internal.compiled_runtime -import org.neo4j.cypher.internal.runtime.InternalExecutionResult +import org.neo4j.cypher.internal.RewindableExecutionResult import org.neo4j.cypher.{CypherTypeException, ExecutionEngineFunSuite} class TypeConversionTest extends ExecutionEngineFunSuite { @@ -45,6 +45,6 @@ class TypeConversionTest extends ExecutionEngineFunSuite { result.toList should equal(List(Map("x" -> x))) } - override def execute(q: String, params: (String, Any)*): InternalExecutionResult = + override def execute(q: String, params: (String, Any)*): RewindableExecutionResult = super.execute(s"cypher runtime=compiled $q", params:_*) } diff --git a/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/queryReduction/CypherReductionSupport.scala b/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/queryReduction/CypherReductionSupport.scala index 9abbefa20609b..9a4ed4b230c58 100644 --- a/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/queryReduction/CypherReductionSupport.scala +++ b/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/queryReduction/CypherReductionSupport.scala @@ -34,7 +34,6 @@ import org.neo4j.cypher.internal.planner.v3_5.spi.{IDPPlannerName, PlanContext, import org.neo4j.cypher.internal.queryReduction.DDmin.Oracle import org.neo4j.cypher.internal.runtime.interpreted.TransactionBoundQueryContext.IndexSearchMonitor import org.neo4j.cypher.internal.runtime.interpreted._ -import org.neo4j.cypher.internal.runtime.{InternalExecutionResult, NormalMode} import org.neo4j.cypher.internal.spi.codegen.GeneratedQueryStructure import org.neo4j.cypher.internal.{CommunityRuntimeFactory, EnterpriseRuntimeContextCreator, MasterCompiler, RewindableExecutionResult} import org.neo4j.cypher.{CypherRuntimeOption, GraphIcing} @@ -113,21 +112,21 @@ trait CypherReductionSupport extends CypherTestSupport with GraphIcing { private val rewriting = PreparatoryRewriting andThen SemanticAnalysis(warn = true).adds(BaseContains[SemanticState]) - def evaluate(query: String, executeBefore: Option[String] = None, enterprise: Boolean = false): InternalExecutionResult = { + def evaluate(query: String, executeBefore: Option[String] = None, enterprise: Boolean = false): RewindableExecutionResult = { val parsingBaseState = queryToParsingBaseState(query, enterprise) val statement = parsingBaseState.statement() produceResult(query, statement, parsingBaseState, executeBefore, enterprise) } - def reduceQuery(query: String, executeBefore: Option[String] = None, enterprise: Boolean = false)(test: Oracle[Try[InternalExecutionResult]]): String = { - val oracle: Oracle[Try[(String, InternalExecutionResult)]] = (tryTuple) => { + def reduceQuery(query: String, executeBefore: Option[String] = None, enterprise: Boolean = false)(test: Oracle[Try[RewindableExecutionResult]]): String = { + val oracle: Oracle[Try[(String, RewindableExecutionResult)]] = (tryTuple) => { val tryResult = tryTuple.map(_._2) test(tryResult) } reduceQueryWithCurrentQueryText(query, executeBefore, enterprise)(oracle) } - def reduceQueryWithCurrentQueryText(query: String, executeBefore: Option[String] = None, enterprise: Boolean = false)(test: Oracle[Try[(String, InternalExecutionResult)]]): String = { + def reduceQueryWithCurrentQueryText(query: String, executeBefore: Option[String] = None, enterprise: Boolean = false)(test: Oracle[Try[(String, RewindableExecutionResult)]]): String = { val parsingBaseState = queryToParsingBaseState(query, enterprise) val statement = parsingBaseState.statement() @@ -153,7 +152,7 @@ trait CypherReductionSupport extends CypherTestSupport with GraphIcing { statement: Statement, parsingBaseState: BaseState, executeBefore: Option[String], - enterprise: Boolean): InternalExecutionResult = { + enterprise: Boolean): RewindableExecutionResult = { val explicitTx = graph.beginTransaction(Transaction.Type.explicit, LoginContext.AUTH_DISABLED) val implicitTx = graph.beginTransaction(Transaction.Type.`implicit`, LoginContext.AUTH_DISABLED) try { @@ -176,7 +175,7 @@ trait CypherReductionSupport extends CypherTestSupport with GraphIcing { parsingBaseState: BaseState, implicitTx: InternalTransaction, enterprise: Boolean - ): InternalExecutionResult = { + ): RewindableExecutionResult = { val neo4jtxContext = contextFactory.newContext(EMBEDDED_CONNECTION, implicitTx, query, EMPTY_MAP) val txContextWrapper = TransactionalContextWrapper(neo4jtxContext) val planContext = TransactionBoundPlanContext(txContextWrapper, devNullLogger) @@ -206,7 +205,8 @@ trait CypherReductionSupport extends CypherTestSupport with GraphIcing { val queryContext = new TransactionBoundQueryContext(txContextWrapper)(CypherReductionSupport.searchMonitor) - RewindableExecutionResult(executionPlan.run(queryContext, NormalMode, ValueConversion.asValues(baseState.extractedParams()))) + val runtimeResult = executionPlan.run(queryContext, doProfile = false, ValueConversion.asValues(baseState.extractedParams())) + RewindableExecutionResult(runtimeResult, queryContext) } private def createContext(query: String, metricsFactory: CachedMetricsFactory, diff --git a/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/queryReduction/CypherReductionSupportTest.scala b/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/queryReduction/CypherReductionSupportTest.scala index 1e3e506070105..3b745ede8cc7f 100644 --- a/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/queryReduction/CypherReductionSupportTest.scala +++ b/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/queryReduction/CypherReductionSupportTest.scala @@ -22,7 +22,7 @@ */ package org.neo4j.cypher.internal.queryReduction -import org.neo4j.cypher.internal.runtime.InternalExecutionResult +import org.neo4j.cypher.internal.RewindableExecutionResult import org.opencypher.v9_0.util.test_helpers.CypherFunSuite import org.opencypher.v9_0.util.ArithmeticException @@ -40,7 +40,7 @@ class CypherReductionSupportTest extends CypherFunSuite with CypherReductionSupp test("removes unnecessary where") { val setup = "CREATE (n {name: \"x\"}) RETURN n" val query = "MATCH (n) WHERE true RETURN n.name" - val reduced = reduceQuery(query, Some(setup)) { (tryResults: Try[InternalExecutionResult]) => + val reduced = reduceQuery(query, Some(setup)) { (tryResults: Try[RewindableExecutionResult]) => tryResults match { case Success(result) => val list = result.toList @@ -69,7 +69,7 @@ class CypherReductionSupportTest extends CypherFunSuite with CypherReductionSupp test("removes unnecessary stuff from faulty query") { val setup = "CREATE (n:Label {name: 0}) RETURN n" val query = s"MATCH (n:Label)-[:X]->(m:Label),(p) WHERE 100/n.name > 34 AND m.name = n.name WITH n.name AS name RETURN name, $$a ORDER BY name SKIP 1 LIMIT 5" - val reduced = reduceQuery(query, Some(setup)) { (tryResults: Try[InternalExecutionResult]) => + val reduced = reduceQuery(query, Some(setup)) { (tryResults: Try[RewindableExecutionResult]) => tryResults match { case Failure(e:ArithmeticException) => if(e.getMessage == "/ by zero" || e.getMessage == "divide by zero") @@ -85,7 +85,7 @@ class CypherReductionSupportTest extends CypherFunSuite with CypherReductionSupp test("removes unnecessary stuff from faulty query with enterprise") { val setup = "CREATE (n:Label {name: 0}) RETURN n" val query = s"MATCH (n:Label)-[:X]->(m:Label),(p) WHERE 100/n.name > 34 AND m.name = n.name WITH n.name AS name RETURN name, $$a ORDER BY name SKIP 1 LIMIT 5" - val reduced = reduceQuery(query, Some(setup), enterprise = true) { (tryResults: Try[InternalExecutionResult]) => + val reduced = reduceQuery(query, Some(setup), enterprise = true) { (tryResults: Try[RewindableExecutionResult]) => tryResults match { case Failure(e:ArithmeticException) => if(e.getMessage == "/ by zero" || e.getMessage == "divide by zero") diff --git a/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/spi/codegen/CompiledExecutionResultTest.scala b/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/spi/codegen/CompiledExecutionResultTest.scala deleted file mode 100644 index c30c5d295d94d..0000000000000 --- a/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/spi/codegen/CompiledExecutionResultTest.scala +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Copyright (c) 2002-2018 "Neo4j," - * Neo4j Sweden AB [http://neo4j.com] - * - * This file is part of Neo4j Enterprise Edition. The included source - * code can be redistributed and/or modified under the terms of the - * GNU AFFERO GENERAL PUBLIC LICENSE Version 3 - * (http://www.fsf.org/licensing/licenses/agpl-3.0.html) with the - * Commons Clause, as found in the associated LICENSE.txt file. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * Neo4j object code can be licensed independently from the source - * under separate terms from the AGPL. Inquiries can be directed to: - * licensing@neo4j.com - * - * More information is also available at: - * https://neo4j.com/licensing/ - */ -package org.neo4j.cypher.internal.spi.codegen - -import java.util - -import org.mockito.ArgumentMatchers._ -import org.mockito.Mockito._ -import org.mockito.invocation.InvocationOnMock -import org.mockito.stubbing.Answer -import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.Completable -import org.neo4j.cypher.internal.executionplan.GeneratedQueryExecution -import org.neo4j.cypher.internal.javacompat.ResultRecord -import org.neo4j.cypher.internal.runtime.compiled.CompiledExecutionResult -import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription -import org.neo4j.cypher.internal.runtime.{ExecutionMode, NormalMode, QueryContext} -import org.opencypher.v9_0.util.TaskCloser -import org.opencypher.v9_0.util.test_helpers.CypherFunSuite -import org.neo4j.cypher.result.QueryResult.QueryResultVisitor -import org.neo4j.function.ThrowingBiConsumer -import org.neo4j.graphdb.NotFoundException -import org.neo4j.graphdb.Result.{ResultRow, ResultVisitor} -import org.neo4j.helpers.collection.Iterators -import org.neo4j.kernel.impl.util.ValueUtils -import org.neo4j.values.AnyValue -import org.neo4j.values.storable._ -import org.neo4j.values.virtual.{ListValue, MapValue} - -import scala.collection.JavaConverters -import scala.collection.JavaConverters._ - -class CompiledExecutionResultTest extends CypherFunSuite { - - test("should return scala objects") { - val result = newCompiledExecutionResult(javaMap("foo" -> "", "bar" -> "")) - - result.columns should equal(List("foo", "bar")) - } - - test("should return scala objects for string") { - val result = newCompiledExecutionResult(javaMap("foo" -> "bar")) - - result.columnAs[String]("foo").toList should equal(List("bar")) - } - - test("should throw if non-existing column") { - val result = newCompiledExecutionResult(javaMap("foo" -> "bar")) - - a [NotFoundException] shouldBe thrownBy(result.columnAs[String]("baz")) - } - - test("should return scala objects for list") { - val result = newCompiledExecutionResult(javaMap("foo" -> javaList(42))) - - result.columnAs[List[Integer]]("foo").toList should equal(List(List(42))) - } - - test("should return scala objects for map") { - val result = newCompiledExecutionResult(javaMap("foo" -> javaMap("key" -> "value"))) - - result.columnAs[Map[String, Any]]("foo").toList should equal(List(Map("key" -> "value"))) - } - - test("should return java objects for string") { - val result = newCompiledExecutionResult(javaMap("foo" -> "bar")) - - Iterators.asList(result.javaColumnAs[String]("foo")) should equal(javaList("bar")) - } - - test("should return java objects for list") { - val result = newCompiledExecutionResult(javaMap("foo" -> javaList(42L))) - - Iterators.asList(result.javaColumnAs[List[Integer]]("foo")) should equal(javaList(javaList(42L))) - } - - test("should return java objects for map") { - val result = newCompiledExecutionResult(javaMap("foo" -> javaMap("key" -> "value"))) - - Iterators.asList(result.javaColumnAs[Map[String, Any]]("foo")) should equal(javaList(javaMap("key" -> "value"))) - } - - test("result should be a scala iterator for string") { - val result = newCompiledExecutionResult(javaMap("foo" -> "bar")) - - result.toList should equal(List(Map("foo" -> "bar"))) - } - - test("result should be a scala iterator for list") { - val result = newCompiledExecutionResult(javaMap("foo" -> javaList(42))) - - result.toList should equal(List(Map("foo" -> List(42)))) - } - - test("result should be a scala iterator for map") { - val result = newCompiledExecutionResult(javaMap("foo" -> javaMap("key" -> "value"))) - - result.toList should equal(List(Map("foo" -> Map("key" -> "value")))) - } - - test("should return a java iterator for string") { - val result = newCompiledExecutionResult(javaMap("foo" -> "bar")) - - Iterators.asList(result.javaIterator) should equal(javaList(javaMap("foo" -> "bar"))) - } - - test("should return a java iterator for list") { - val result = newCompiledExecutionResult(javaMap("foo" -> javaList(42L))) - - Iterators.asList(result.javaIterator) should equal(javaList(javaMap("foo" -> javaList(42L)))) - } - - test("should return a java iterator for map") { - val result = newCompiledExecutionResult(javaMap("foo" -> javaMap("key" -> "value"))) - - Iterators.asList(result.javaIterator) should equal(javaList(javaMap("foo" -> javaMap("key" -> "value")))) - } - - test("javaIterator hasNext should not call accept if results already consumed") { - // given - var timesCalled = 0 - val result = newCompiledExecutionResult(assertion = () => { - // then - timesCalled should equal(0) - timesCalled += 1 - }) - - // when - result.accept(new ResultVisitor[Exception] { - override def visit(row: ResultRow): Boolean = { - false - } - }) - } - - test("close should work after result is consumed") { - // given - val result = newCompiledExecutionResult(javaMap("a" -> "1", "b" -> "2")) - - // when - result.accept(new ResultVisitor[Exception] { - override def visit(row: ResultRow): Boolean = { - true - } - }) - - result.close() - - // then - // call of close actually worked - } - - private def newCompiledExecutionResult(row: util.Map[String, Any] = new util.HashMap(), - taskCloser: TaskCloser = new TaskCloser, - assertion: () => Unit = () => {}) = { - val noCompiledCode: GeneratedQueryExecution = new GeneratedQueryExecution { - override def setCompletable(closeable: Completable){} - - override def fieldNames(): Array[String] = row.keySet().toArray(new Array[String](row.size())) - - override def executionMode(): ExecutionMode = NormalMode - override def accept[E <: Exception](visitor: QueryResultVisitor[E]): Unit = { - try { - val fields = new Array[AnyValue](row.size()) - var i = 0 - row.asScala.foreach { case (k, v) => - fields(i) = ValueUtils.of(v) - i += 1 - } - visitor.visit(new ResultRecord(fields)) - assertion() - } finally { - taskCloser.close(success = true) - } - } - override def executionPlanDescription(): InternalPlanDescription = ??? - } - - val context = mock[QueryContext] - when(context.asObject(any())).thenAnswer(new Answer[AnyRef] { - override def answer(invocationOnMock: InvocationOnMock): AnyRef = - toObjectConverter(invocationOnMock.getArguments()(0)) - }) - when(context.isGraphKernelResultValue(any())).thenReturn(false) - - new CompiledExecutionResult(taskCloser, context, noCompiledCode, null) - } - - import JavaConverters._ - private def toObjectConverter(a: AnyRef): AnyRef = a match { - case Values.NO_VALUE => null - case s: TextValue => s.stringValue() - case b: BooleanValue => Boolean.box(b.booleanValue()) - case f: FloatingPointValue => Double.box(f.doubleValue()) - case i: IntegralValue => Long.box(i.longValue()) - case l: ListValue => - val list = new util.ArrayList[AnyRef] - l.iterator().asScala.foreach(a => list.add(toObjectConverter(a))) - list - case m: MapValue => - val map = new util.HashMap[String, AnyRef]() - m.foreach(new ThrowingBiConsumer[String, AnyValue, RuntimeException] { - override def accept(t: String, u: AnyValue): Unit = map.put(t, toObjectConverter(u)) - }) - map - } - - private def javaList[T](elements: T*): util.List[T] = elements.toList.asJava - - private def javaMap[K, V](pairs: (K, V)*): util.Map[K, V] = pairs.toMap.asJava -} diff --git a/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/spi/codegen/GeneratedMethodStructureTest.scala b/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/spi/codegen/GeneratedMethodStructureTest.scala index 1517f96b3e0c7..c7a4d94f0a138 100644 --- a/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/spi/codegen/GeneratedMethodStructureTest.scala +++ b/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/spi/codegen/GeneratedMethodStructureTest.scala @@ -28,7 +28,7 @@ import org.neo4j.codegen.bytecode.ByteCode import org.neo4j.codegen.source.SourceCode import org.neo4j.codegen.{CodeGenerationStrategy, CodeGenerator, Expression, MethodDeclaration} import org.neo4j.cypher.internal.codegen.QueryExecutionTracer -import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.{Completable, Provider} +import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.Provider import org.opencypher.v9_0.frontend.helpers._ import org.opencypher.v9_0.ast.semantics.SemanticTable import org.neo4j.cypher.internal.runtime.compiled.codegen.CodeGenContext @@ -235,7 +235,6 @@ class GeneratedMethodStructureTest extends CypherFunSuite { description = body.field(typeRef[Provider[InternalPlanDescription]], "description"), tracer = body.field(typeRef[QueryExecutionTracer], "tracer"), params = body.field(typeRef[util.Map[String, Object]], "params"), - closeable = body.field(typeRef[Completable], "closeable"), queryContext = body.field(typeRef[QueryContext], "queryContext"), skip = body.field(typeRef[Boolean], "skip"), cursors = body.field(typeRef[CursorFactory], "cursors"), diff --git a/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/spi/codegen/ir/BuildProbeTableInstructionsTest.scala b/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/spi/codegen/ir/BuildProbeTableInstructionsTest.scala index f345f2d02720e..42c530c7ff55b 100644 --- a/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/spi/codegen/ir/BuildProbeTableInstructionsTest.scala +++ b/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/spi/codegen/ir/BuildProbeTableInstructionsTest.scala @@ -220,7 +220,7 @@ class BuildProbeTableInstructionsTest extends CypherFunSuite with CodeGenSugar { override def next() = inner.next() } - private def runTest(buildInstruction: BuildProbeTable, nodes: Set[Variable]): List[Map[String, Object]] = { + private def runTest(buildInstruction: BuildProbeTable, nodes: Set[Variable]): Seq[Map[String, Object]] = { val instructions = buildProbeTableWithTwoAllNodeScans(buildInstruction, nodes) val ids: Map[String, Id] = instructions.flatMap(_.allOperatorIds.map(id => id -> Id.INVALID_ID)).toMap evaluate(instructions, queryContext, Seq(resultRowKey), EMPTY_MAP, ids) diff --git a/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/spi/codegen/ir/CodeGenSugar.scala b/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/spi/codegen/ir/CodeGenSugar.scala index 3fb9b6013bf62..f2c1511283efe 100644 --- a/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/spi/codegen/ir/CodeGenSugar.scala +++ b/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/spi/codegen/ir/CodeGenSugar.scala @@ -25,26 +25,23 @@ package org.neo4j.cypher.internal.compiled_runtime.v3_5.codegen.ir import java.util.concurrent.atomic.AtomicInteger import org.mockito.Mockito._ +import org.neo4j.cypher.internal.RewindableExecutionResult import org.neo4j.cypher.internal.codegen.QueryExecutionTracer -import org.neo4j.cypher.internal.runtime.compiled.ExecutionPlanBuilder.tracer -import org.neo4j.cypher.internal.runtime.compiled.codegen._ -import org.neo4j.cypher.internal.runtime.compiled.codegen.ir.Instruction +import org.neo4j.cypher.internal.codegen.profiling.ProfilingTracer import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.Provider import org.neo4j.cypher.internal.compiler.v3_5.planner.LogicalPlanConstructionTestSupport import org.neo4j.cypher.internal.executionplan.{GeneratedQuery, GeneratedQueryExecution} -import org.opencypher.v9_0.ast.semantics.SemanticTable import org.neo4j.cypher.internal.planner.v3_5.spi.{CostBasedPlannerName, GraphStatistics, PlanContext} +import org.neo4j.cypher.internal.runtime._ +import org.neo4j.cypher.internal.runtime.compiled.codegen._ +import org.neo4j.cypher.internal.runtime.compiled.codegen.ir.Instruction import org.neo4j.cypher.internal.runtime.compiled.{CompiledExecutionResult, CompiledPlan} import org.neo4j.cypher.internal.runtime.interpreted.TransactionBoundQueryContext.IndexSearchMonitor import org.neo4j.cypher.internal.runtime.interpreted.{TransactionBoundQueryContext, TransactionalContextWrapper} -import org.neo4j.cypher.internal.runtime.planDescription.InternalPlanDescription -import org.neo4j.cypher.internal.runtime.{ExecutionMode, InternalExecutionResult, NormalMode, QueryContext} import org.neo4j.cypher.internal.spi.codegen.GeneratedQueryStructure -import org.opencypher.v9_0.util.TaskCloser -import org.opencypher.v9_0.util.attribution.Id import org.neo4j.cypher.internal.v3_5.logical.plans.LogicalPlan +import org.neo4j.cypher.result.QueryProfile import org.neo4j.graphdb.GraphDatabaseService -import org.neo4j.graphdb.Result.{ResultRow, ResultVisitor} import org.neo4j.internal.kernel.api.Transaction.Type import org.neo4j.kernel.GraphDatabaseQueryService import org.neo4j.kernel.api.security.AnonymousContext @@ -54,6 +51,9 @@ import org.neo4j.kernel.impl.query.clientconnection.ClientConnectionInfo import org.neo4j.time.Clocks import org.neo4j.values.virtual.MapValue import org.neo4j.values.virtual.VirtualValues.EMPTY_MAP +import org.opencypher.v9_0.ast.semantics.SemanticTable +import org.opencypher.v9_0.util.TaskCloser +import org.opencypher.v9_0.util.attribution.Id import org.scalatest.mock.MockitoSugar trait CodeGenSugar extends MockitoSugar with LogicalPlanConstructionTestSupport { @@ -68,15 +68,12 @@ trait CodeGenSugar extends MockitoSugar with LogicalPlanConstructionTestSupport .generate(plan, context, semanticTable, CostBasedPlannerName.default, readOnly = true, new StubCardinalities) } - def compileAndExecute(plan: LogicalPlan, - graphDb: GraphDatabaseQueryService, - mode: ExecutionMode = NormalMode) = { - executeCompiled(compile(plan), graphDb, mode) + def compileAndProfile(plan: LogicalPlan, graphDb: GraphDatabaseQueryService): RewindableExecutionResult = { + profileCompiled(compile(plan), graphDb) } - def executeCompiled(plan: CompiledPlan, - graphDb: GraphDatabaseQueryService, - mode: ExecutionMode = NormalMode): InternalExecutionResult = { + def profileCompiled(plan: CompiledPlan, + graphDb: GraphDatabaseQueryService): RewindableExecutionResult = { val tx = graphDb.beginTransaction(Type.explicit, AnonymousContext.read()) var transactionalContext: TransactionalContextWrapper = null try { @@ -86,11 +83,10 @@ trait CodeGenSugar extends MockitoSugar with LogicalPlanConstructionTestSupport contextFactory.newContext(ClientConnectionInfo.EMBEDDED_CONNECTION, tx, "no query text exists for this test", EMPTY_MAP)) val queryContext = new TransactionBoundQueryContext(transactionalContext)(mock[IndexSearchMonitor]) - val result = plan - .executionResultBuilder(queryContext, mode, tracer(mode, queryContext), EMPTY_MAP, new TaskCloser) + val tracer = Some(new ProfilingTracer(queryContext.transactionalContext.kernelStatisticProvider)) + val result = plan.executionResultBuilder(queryContext, ProfileMode, tracer, EMPTY_MAP) tx.success() - result.size - result + RewindableExecutionResult(result, queryContext) } finally { transactionalContext.close(true) tx.close() @@ -101,22 +97,9 @@ trait CodeGenSugar extends MockitoSugar with LogicalPlanConstructionTestSupport qtx: QueryContext = mockQueryContext(), columns: Seq[String] = Seq.empty, params: MapValue = EMPTY_MAP, - operatorIds: Map[String, Id] = Map.empty): List[Map[String, Object]] = { + operatorIds: Map[String, Id] = Map.empty): Seq[Map[String, Object]] = { val clazz = compile(instructions, columns, operatorIds) - val result = newInstance(clazz, queryContext = qtx, params = params) - evaluate(result) - } - - def evaluate(result: InternalExecutionResult): List[Map[String, Object]] = { - var rows = List.empty[Map[String, Object]] - val columns: List[String] = result.columns - result.accept(new ResultVisitor[RuntimeException] { - override def visit(row: ResultRow): Boolean = { - rows = rows :+ columns.map(key => (key, row.get(key))).toMap - true - } - }) - rows + newInstance(clazz, queryContext = qtx, params = params).toList } def codeGenConfiguration = CodeGenConfiguration(mode = ByteCodeMode) @@ -135,12 +118,17 @@ trait CodeGenSugar extends MockitoSugar with LogicalPlanConstructionTestSupport queryContext: QueryContext = mockQueryContext(), graphdb: GraphDatabaseService = null, executionMode: ExecutionMode = null, - provider: Provider[InternalPlanDescription] = null, - queryExecutionTracer: QueryExecutionTracer = QueryExecutionTracer.NONE, - params: MapValue = EMPTY_MAP): InternalExecutionResult = { + tracer: Option[ProfilingTracer] = None, + params: MapValue = EMPTY_MAP): RewindableExecutionResult = { + val generated = clazz.execute(queryContext, - executionMode, provider, queryExecutionTracer, params) - new CompiledExecutionResult(taskCloser, queryContext, generated, provider) + executionMode, + Provider.NULL(), + tracer.getOrElse(QueryExecutionTracer.NONE), + params) + + val runtimeResult = new CompiledExecutionResult(queryContext, generated, tracer.getOrElse(QueryProfile.NONE)) + RewindableExecutionResult(runtimeResult, queryContext) } def insertStatic(clazz: Class[GeneratedQueryExecution], mappings: (String, Id)*) = mappings.foreach { diff --git a/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/spi/codegen/ir/CompiledProfilingTest.scala b/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/spi/codegen/ir/CompiledProfilingTest.scala index f60895895380f..605f62632e9cc 100644 --- a/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/spi/codegen/ir/CompiledProfilingTest.scala +++ b/enterprise/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/spi/codegen/ir/CompiledProfilingTest.scala @@ -85,7 +85,7 @@ class CompiledProfilingTest extends CypherFunSuite with CodeGenSugar { // when val tracer = new ProfilingTracer(transactionalContext.kernelStatisticProvider) - newInstance(compiled, queryContext = queryContext, provider = provider, queryExecutionTracer = tracer).size + newInstance(compiled, queryContext = queryContext, tracer = Some(tracer)).size // then tracer.dbHitsOf(id1) should equal(3) @@ -115,7 +115,7 @@ class CompiledProfilingTest extends CypherFunSuite with CodeGenSugar { val plan = plans.ProduceResult(projection, List("foo")) // when - val result = compileAndExecute(plan, graphDb, mode = ProfileMode) + val result = compileAndProfile(plan, graphDb) val description = result.executionPlanDescription() // then diff --git a/enterprise/cypher/morsel-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/vectorized/Dispatcher.scala b/enterprise/cypher/morsel-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/vectorized/Dispatcher.scala index 3f989dffe8687..63a9091555cff 100644 --- a/enterprise/cypher/morsel-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/vectorized/Dispatcher.scala +++ b/enterprise/cypher/morsel-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/vectorized/Dispatcher.scala @@ -26,14 +26,12 @@ import org.neo4j.cypher.internal.runtime.QueryContext import org.neo4j.cypher.internal.runtime.parallel.{Scheduler, SchedulerTracer, SingleThreadScheduler} import org.neo4j.cypher.result.QueryResult.QueryResultVisitor import org.neo4j.values.virtual.MapValue -import org.opencypher.v9_0.util.TaskCloser class Dispatcher(morselSize: Int, scheduler: Scheduler) { def execute[E <: Exception](operators: Pipeline, queryContext: QueryContext, params: MapValue, - taskCloser: TaskCloser, schedulerTracer: SchedulerTracer) (visitor: QueryResultVisitor[E]): Unit = { val leaf = getLeaf(operators) @@ -42,14 +40,8 @@ class Dispatcher(morselSize: Int, scheduler: Scheduler) { val initialTask = leaf.init(MorselExecutionContext.EMPTY, queryContext, state) val queryExecution = scheduler.execute(initialTask, schedulerTracer) val maybeError = queryExecution.await() - maybeError match { - case Some(error) => - taskCloser.close(false) - throw error - - case None => - taskCloser.close(true) - } + if (maybeError.isDefined) + throw maybeError.get } private def getLeaf(pipeline: Pipeline): StreamingPipeline = { diff --git a/enterprise/cypher/slotted-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/slotted/SlottedExecutionResultBuilderFactory.scala b/enterprise/cypher/slotted-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/slotted/SlottedExecutionResultBuilderFactory.scala index d361469db2522..f79b7d1f03342 100644 --- a/enterprise/cypher/slotted-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/slotted/SlottedExecutionResultBuilderFactory.scala +++ b/enterprise/cypher/slotted-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/slotted/SlottedExecutionResultBuilderFactory.scala @@ -20,11 +20,12 @@ package org.neo4j.cypher.internal.runtime.slotted import org.neo4j.cypher.internal.compatibility.v3_5.runtime.PhysicalPlanningAttributes.SlotConfigurations -import org.neo4j.cypher.internal.compatibility.v3_5.runtime.{ClosingQueryResultRecordIterator, ResultIterator} +import org.neo4j.cypher.internal.compatibility.v3_5.runtime.IteratorBasedResult import org.neo4j.cypher.internal.compatibility.v3_5.runtime.executionplan.{BaseExecutionResultBuilderFactory, ExecutionResultBuilder, PipeInfo} import org.neo4j.cypher.internal.runtime.interpreted.ExecutionContext import org.neo4j.cypher.internal.runtime.interpreted.pipes.Pipe import org.neo4j.cypher.internal.v3_5.logical.plans.LogicalPlan +import org.neo4j.cypher.result.QueryResult import org.neo4j.values.virtual.MapValue import scala.collection.mutable @@ -40,15 +41,13 @@ class SlottedExecutionResultBuilderFactory(pipe: Pipe, new SlottedExecutionWorkflowBuilder() class SlottedExecutionWorkflowBuilder() extends BaseExecutionWorkflowBuilder { - override protected def createQueryState(params: MapValue) = { + override protected def createQueryState(params: MapValue): SlottedQueryState = { new SlottedQueryState(queryContext, externalResource, params, pipeDecorator, triadicState = mutable.Map.empty, repeatableReads = mutable.Map.empty) } - override def buildResultIterator(results: Iterator[ExecutionContext], readOnly: Boolean): ResultIterator = { - val closingIterator = new ClosingQueryResultRecordIterator(results, taskCloser, exceptionDecorator) - val resultIterator = if (!readOnly) closingIterator.toEager else closingIterator - resultIterator + override def buildResultIterator(results: Iterator[ExecutionContext], readOnly: Boolean): IteratorBasedResult = { + IteratorBasedResult(results, Some(results.asInstanceOf[Iterator[QueryResult.Record]])) } } } diff --git a/enterprise/cypher/slotted-runtime/src/test/scala/org/neo4j/cypher/internal/runtime/slotted/SlottedPipeBuilderTest.scala b/enterprise/cypher/slotted-runtime/src/test/scala/org/neo4j/cypher/internal/runtime/slotted/SlottedPipeBuilderTest.scala index 1424267dc1035..c73eaf0c648bd 100644 --- a/enterprise/cypher/slotted-runtime/src/test/scala/org/neo4j/cypher/internal/runtime/slotted/SlottedPipeBuilderTest.scala +++ b/enterprise/cypher/slotted-runtime/src/test/scala/org/neo4j/cypher/internal/runtime/slotted/SlottedPipeBuilderTest.scala @@ -57,7 +57,7 @@ class SlottedPipeBuilderTest extends CypherFunSuite with LogicalPlanningTestSupp val logicalPlan = slottedRewriter(beforeRewrite, physicalPlan.slotConfigurations) val converters = new ExpressionConverters(CommunityExpressionConverter, SlottedExpressionConverters) val executionPlanBuilder = new PipeExecutionPlanBuilder(SlottedPipeBuilder.Factory(physicalPlan), converters) - val context = PipeExecutionBuilderContext(table, true, new StubCardinalities) + val context = PipeExecutionBuilderContext(table, true) executionPlanBuilder.build(logicalPlan)(context, planContext) }