From 2694e14b67a1857be3d2f807b7a6a1733bb9ff40 Mon Sep 17 00:00:00 2001 From: Tobias Lindaaker Date: Thu, 5 Jan 2017 15:57:57 +0100 Subject: [PATCH] Provide information about which indexes a query uses Only works with Cypher version 3.2, support in older versions would require changes to those versions and new releases. --- .../org/neo4j/function/FailableConsumer.java | 27 ++++ .../java/org/neo4j/function/Predicates.java | 9 +- .../java/org/neo4j/function/Suppliers.java | 17 ++- .../org/neo4j/function/ThrowingPredicate.java | 20 +++ .../org/neo4j/function/ThrowingSupplier.java | 20 +++ .../v3_2/BuildCompiledExecutionPlan.scala | 2 + .../v3_2/BuildInterpretedExecutionPlan.scala | 2 + .../compiler/v3_2/codegen/CodeGenerator.scala | 2 +- .../v3_2/executionplan/ExecutionPlan.scala | 12 +- .../executionplan/ExecutionPlanBuilder.scala | 3 +- .../planner/logical/plans/LogicalPlan.scala | 25 +++- .../compatibility/v2_3/Compatibility.scala | 6 +- .../compatibility/v3_1/Compatibility.scala | 6 +- .../compatibility/v3_2/Compatibility.scala | 13 +- ...r.java => FailableConcurrentTransfer.java} | 27 +++- .../org/neo4j/kernel/api/ExecutingQuery.java | 29 +++- .../java/org/neo4j/kernel/api/IndexUsage.java | 93 ++++++++++++ .../neo4j/kernel/api/ExecutingQueryTest.java | 7 +- .../test/rule/concurrent/ThreadingRule.java | 40 +++++- .../builtinprocs/QueryStatusResult.java | 3 + .../ListQueriesProcedureTest.java | 134 +++++++++++++++++- 21 files changed, 458 insertions(+), 39 deletions(-) create mode 100644 community/common/src/main/java/org/neo4j/function/FailableConsumer.java rename community/kernel/src/main/java/org/neo4j/helpers/{ConcurrentTransfer.java => FailableConcurrentTransfer.java} (70%) create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/api/IndexUsage.java diff --git a/community/common/src/main/java/org/neo4j/function/FailableConsumer.java b/community/common/src/main/java/org/neo4j/function/FailableConsumer.java new file mode 100644 index 0000000000000..7b3c9e9e7b93c --- /dev/null +++ b/community/common/src/main/java/org/neo4j/function/FailableConsumer.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.function; + +import java.util.function.Consumer; + +public interface FailableConsumer extends Consumer +{ + void fail( Exception failure ); +} diff --git a/community/common/src/main/java/org/neo4j/function/Predicates.java b/community/common/src/main/java/org/neo4j/function/Predicates.java index 0b0bd57037c62..fc79ef55600f9 100644 --- a/community/common/src/main/java/org/neo4j/function/Predicates.java +++ b/community/common/src/main/java/org/neo4j/function/Predicates.java @@ -32,6 +32,9 @@ import java.util.function.Supplier; import javax.annotation.Nonnull; +import static org.neo4j.function.ThrowingPredicate.throwingPredicate; +import static org.neo4j.function.ThrowingSupplier.throwingSupplier; + /** * Constructors for basic {@link Predicate} types */ @@ -147,7 +150,7 @@ public static TYPE await( Supplier supplier, Predicate predic public static TYPE await( Supplier supplier, Predicate predicate, long timeout, TimeUnit timeoutUnit ) throws TimeoutException { - return awaitEx( supplier::get, predicate::test, timeout, timeoutUnit ); + return awaitEx( throwingSupplier( supplier ), throwingPredicate( predicate ), timeout, timeoutUnit ); } public static TYPE awaitEx( ThrowingSupplier supplier, @@ -159,8 +162,8 @@ public static TYPE awaitEx( ThrowingSupplier return composed.lastInput(); } - public static TYPE awaitEx( ThrowingSupplier supplier, - ThrowingPredicate predicate, long timeout, TimeUnit timeoutUnit ) + public static TYPE awaitEx( ThrowingSupplier supplier, + ThrowingPredicate predicate, long timeout, TimeUnit timeoutUnit ) throws TimeoutException, EXCEPTION { Suppliers.ThrowingCapturingSupplier composed = Suppliers.compose( supplier, predicate ); diff --git a/community/common/src/main/java/org/neo4j/function/Suppliers.java b/community/common/src/main/java/org/neo4j/function/Suppliers.java index d1ce4af20bd64..013ca1c63dbed 100644 --- a/community/common/src/main/java/org/neo4j/function/Suppliers.java +++ b/community/common/src/main/java/org/neo4j/function/Suppliers.java @@ -118,8 +118,9 @@ public T get() }; } - public static ThrowingCapturingSupplier compose( final ThrowingSupplier input, - final ThrowingPredicate predicate ) + public static ThrowingCapturingSupplier compose( + final ThrowingSupplier input, + final ThrowingPredicate predicate ) { return new ThrowingCapturingSupplier<>( input, predicate ); } @@ -132,12 +133,12 @@ public static BooleanSupplier untilTimeExpired( long duration, TimeUnit unit ) static class ThrowingCapturingSupplier implements ThrowingSupplier { - private final ThrowingSupplier input; - private final ThrowingPredicate predicate; + private final ThrowingSupplier input; + private final ThrowingPredicate predicate; private T current; - ThrowingCapturingSupplier( ThrowingSupplier input, ThrowingPredicate predicate ) + ThrowingCapturingSupplier( ThrowingSupplier input, ThrowingPredicate predicate ) { this.input = input; this.predicate = predicate; @@ -154,5 +155,11 @@ public Boolean get() throws E current = input.get(); return predicate.test( current ); } + + @Override + public String toString() + { + return String.format( "%s on %s", predicate, input ); + } } } diff --git a/community/common/src/main/java/org/neo4j/function/ThrowingPredicate.java b/community/common/src/main/java/org/neo4j/function/ThrowingPredicate.java index 898401e5901a6..91ae61ec756bb 100644 --- a/community/common/src/main/java/org/neo4j/function/ThrowingPredicate.java +++ b/community/common/src/main/java/org/neo4j/function/ThrowingPredicate.java @@ -19,6 +19,8 @@ */ package org.neo4j.function; +import java.util.function.Predicate; + /** * Represents a predicate (boolean-valued function) of one argument. * @@ -35,4 +37,22 @@ public interface ThrowingPredicate * @throws E an exception if the predicate fails */ boolean test( T t ) throws E; + + static ThrowingPredicate throwingPredicate( Predicate predicate ) + { + return new ThrowingPredicate() + { + @Override + public boolean test( TYPE value ) + { + return predicate.test( value ); + } + + @Override + public String toString() + { + return predicate.toString(); + } + }; + } } diff --git a/community/common/src/main/java/org/neo4j/function/ThrowingSupplier.java b/community/common/src/main/java/org/neo4j/function/ThrowingSupplier.java index f1cab6b5cc872..bd0ed55d6e134 100644 --- a/community/common/src/main/java/org/neo4j/function/ThrowingSupplier.java +++ b/community/common/src/main/java/org/neo4j/function/ThrowingSupplier.java @@ -19,6 +19,8 @@ */ package org.neo4j.function; +import java.util.function.Supplier; + /** * Represents a supplier of results, that may throw an exception. * @@ -34,4 +36,22 @@ public interface ThrowingSupplier * @throws E an exception if the function fails */ T get() throws E; + + static ThrowingSupplier throwingSupplier( Supplier supplier ) + { + return new ThrowingSupplier() + { + @Override + public TYPE get() + { + return supplier.get(); + } + + @Override + public String toString() + { + return supplier.toString(); + } + }; + } } diff --git a/community/cypher/cypher-compiler-3.2/src/main/scala/org/neo4j/cypher/internal/compiler/v3_2/BuildCompiledExecutionPlan.scala b/community/cypher/cypher-compiler-3.2/src/main/scala/org/neo4j/cypher/internal/compiler/v3_2/BuildCompiledExecutionPlan.scala index 5fd3e83102dbe..f6dd8212eefe7 100644 --- a/community/cypher/cypher-compiler-3.2/src/main/scala/org/neo4j/cypher/internal/compiler/v3_2/BuildCompiledExecutionPlan.scala +++ b/community/cypher/cypher-compiler-3.2/src/main/scala/org/neo4j/cypher/internal/compiler/v3_2/BuildCompiledExecutionPlan.scala @@ -81,6 +81,8 @@ object BuildCompiledExecutionPlan extends Phase { override def runtimeUsed = CompiledRuntimeName override def notifications(planContext: PlanContext): Seq[InternalNotification] = Seq.empty + + override def plannedIndexUsage: Seq[IndexUsage] = compiled.plannedIndexUsage } diff --git a/community/cypher/cypher-compiler-3.2/src/main/scala/org/neo4j/cypher/internal/compiler/v3_2/BuildInterpretedExecutionPlan.scala b/community/cypher/cypher-compiler-3.2/src/main/scala/org/neo4j/cypher/internal/compiler/v3_2/BuildInterpretedExecutionPlan.scala index 731886cc72f3c..a3b129bc6d727 100644 --- a/community/cypher/cypher-compiler-3.2/src/main/scala/org/neo4j/cypher/internal/compiler/v3_2/BuildInterpretedExecutionPlan.scala +++ b/community/cypher/cypher-compiler-3.2/src/main/scala/org/neo4j/cypher/internal/compiler/v3_2/BuildInterpretedExecutionPlan.scala @@ -62,6 +62,8 @@ object BuildInterpretedExecutionPlan extends Phase { override def runtimeUsed = InterpretedRuntimeName override def notifications(planContext: PlanContext) = checkForNotifications(pipe, planContext, context.config) + + override def plannedIndexUsage = logicalPlan.indexUsage } from.copy(maybeExecutionPlan = Some(execPlan)) diff --git a/community/cypher/cypher-compiler-3.2/src/main/scala/org/neo4j/cypher/internal/compiler/v3_2/codegen/CodeGenerator.scala b/community/cypher/cypher-compiler-3.2/src/main/scala/org/neo4j/cypher/internal/compiler/v3_2/codegen/CodeGenerator.scala index 77cdbfd84412d..e35dc45a1ff61 100644 --- a/community/cypher/cypher-compiler-3.2/src/main/scala/org/neo4j/cypher/internal/compiler/v3_2/codegen/CodeGenerator.scala +++ b/community/cypher/cypher-compiler-3.2/src/main/scala/org/neo4j/cypher/internal/compiler/v3_2/codegen/CodeGenerator.scala @@ -72,7 +72,7 @@ class CodeGenerator(val structure: CodeStructure[GeneratedQuery], clock: Clock, } } - CompiledPlan(updating = false, None, fp, plannerName, description, res.columns, builder) + CompiledPlan(updating = false, None, fp, plannerName, description, res.columns, builder, plan.indexUsage) case _ => throw new CantCompileQueryException("Can only compile plans with ProduceResult on top") } diff --git a/community/cypher/cypher-compiler-3.2/src/main/scala/org/neo4j/cypher/internal/compiler/v3_2/executionplan/ExecutionPlan.scala b/community/cypher/cypher-compiler-3.2/src/main/scala/org/neo4j/cypher/internal/compiler/v3_2/executionplan/ExecutionPlan.scala index a893bfe932848..2c1b269615050 100644 --- a/community/cypher/cypher-compiler-3.2/src/main/scala/org/neo4j/cypher/internal/compiler/v3_2/executionplan/ExecutionPlan.scala +++ b/community/cypher/cypher-compiler-3.2/src/main/scala/org/neo4j/cypher/internal/compiler/v3_2/executionplan/ExecutionPlan.scala @@ -19,7 +19,7 @@ */ package org.neo4j.cypher.internal.compiler.v3_2.executionplan -import org.neo4j.cypher.internal.compiler.v3_2.spi.{PlanContext, GraphStatistics, QueryContext} +import org.neo4j.cypher.internal.compiler.v3_2.spi.{GraphStatistics, PlanContext, QueryContext} import org.neo4j.cypher.internal.compiler.v3_2.{ExecutionMode, PlannerName, RuntimeName} import org.neo4j.cypher.internal.frontend.v3_2.notification.InternalNotification @@ -30,4 +30,14 @@ abstract class ExecutionPlan { def isStale(lastTxId: () => Long, statistics: GraphStatistics): Boolean def runtimeUsed: RuntimeName def notifications(planContext: PlanContext): Seq[InternalNotification] + def plannedIndexUsage: Seq[IndexUsage] = Seq.empty } + +sealed trait IndexUsage { + def identifier:String +} + +final case class SchemaIndexSeekUsage(identifier: String, label: String, propertyKey: String) extends IndexUsage +final case class SchemaIndexScanUsage(identifier: String, label: String, propertyKey: String) extends IndexUsage +final case class LegacyNodeIndexUsage(identifier: String, index: String) extends IndexUsage +final case class LegacyRelationshipIndexUsage(identifier: String, index: String) extends IndexUsage diff --git a/community/cypher/cypher-compiler-3.2/src/main/scala/org/neo4j/cypher/internal/compiler/v3_2/executionplan/ExecutionPlanBuilder.scala b/community/cypher/cypher-compiler-3.2/src/main/scala/org/neo4j/cypher/internal/compiler/v3_2/executionplan/ExecutionPlanBuilder.scala index d0489849cdfe1..d87c53cf30b0d 100644 --- a/community/cypher/cypher-compiler-3.2/src/main/scala/org/neo4j/cypher/internal/compiler/v3_2/executionplan/ExecutionPlanBuilder.scala +++ b/community/cypher/cypher-compiler-3.2/src/main/scala/org/neo4j/cypher/internal/compiler/v3_2/executionplan/ExecutionPlanBuilder.scala @@ -45,7 +45,8 @@ case class CompiledPlan(updating: Boolean, plannerUsed: PlannerName, planDescription: InternalPlanDescription, columns: Seq[String], - executionResultBuilder: RunnablePlan ) + executionResultBuilder: RunnablePlan, + plannedIndexUsage: Seq[IndexUsage] = Seq.empty ) case class PipeInfo(pipe: Pipe, updating: Boolean, diff --git a/community/cypher/cypher-compiler-3.2/src/main/scala/org/neo4j/cypher/internal/compiler/v3_2/planner/logical/plans/LogicalPlan.scala b/community/cypher/cypher-compiler-3.2/src/main/scala/org/neo4j/cypher/internal/compiler/v3_2/planner/logical/plans/LogicalPlan.scala index 7b744735beb9a..37536ff4141ca 100644 --- a/community/cypher/cypher-compiler-3.2/src/main/scala/org/neo4j/cypher/internal/compiler/v3_2/planner/logical/plans/LogicalPlan.scala +++ b/community/cypher/cypher-compiler-3.2/src/main/scala/org/neo4j/cypher/internal/compiler/v3_2/planner/logical/plans/LogicalPlan.scala @@ -22,10 +22,11 @@ package org.neo4j.cypher.internal.compiler.v3_2.planner.logical.plans import java.lang.reflect.Method import org.neo4j.cypher.internal.compiler.v3_2.commands.QueryExpression +import org.neo4j.cypher.internal.compiler.v3_2.executionplan._ import org.neo4j.cypher.internal.compiler.v3_2.planner.{CardinalityEstimation, PlannerQuery} import org.neo4j.cypher.internal.frontend.v3_2.Foldable._ import org.neo4j.cypher.internal.frontend.v3_2.Rewritable._ -import org.neo4j.cypher.internal.frontend.v3_2.ast.Expression +import org.neo4j.cypher.internal.frontend.v3_2.ast.{Expression, NodeByIdentifiedIndex, NodeByIndexQuery, RelationshipByIdentifiedIndex, RelationshipByIndexQuery} import org.neo4j.cypher.internal.frontend.v3_2.{InternalException, Rewritable} import org.neo4j.cypher.internal.ir.v3_2.{IdName, Strictness} @@ -113,6 +114,26 @@ abstract class LogicalPlan def debugId: String = f"0x${hashCode()}%08x" def flatten: Seq[LogicalPlan] = Flattener.create(this) + + def indexUsage: Seq[IndexUsage] = { + import org.neo4j.cypher.internal.frontend.v3_2.Foldable._ + this.fold(Seq.empty[IndexUsage]) { + case NodeIndexSeek(idName, label, propertyKey, _, _) => + (acc) => acc :+ SchemaIndexSeekUsage(idName.name, label.name, propertyKey.name) + case NodeUniqueIndexSeek(idName, label, propertyKey, _, _) => + (acc) => acc :+ SchemaIndexSeekUsage(idName.name, label.name, propertyKey.name) + case NodeIndexScan(idName, label, propertyKey, _) => + (acc) => acc :+ SchemaIndexScanUsage(idName.name, label.name, propertyKey.name) + case LegacyNodeIndexSeek(idName, NodeByIdentifiedIndex(_, index, _, _), _) => + (acc) => acc :+ LegacyNodeIndexUsage(idName.name, index) + case LegacyNodeIndexSeek(idName, NodeByIndexQuery(_, index, _), _) => + (acc) => acc :+ LegacyNodeIndexUsage(idName.name, index) + case LegacyRelationshipIndexSeek(idName, RelationshipByIdentifiedIndex(_, index, _, _), _) => + (acc) => acc :+ LegacyRelationshipIndexUsage(idName.name, index) + case LegacyRelationshipIndexSeek(idName, RelationshipByIndexQuery(_, index, _), _) => + (acc) => acc :+ LegacyRelationshipIndexUsage(idName.name, index) + } + } } abstract class LogicalLeafPlan extends LogicalPlan with LazyLogicalPlan { @@ -135,4 +156,4 @@ case object Flattener extends TreeBuilder[Seq[LogicalPlan]] { override protected def build(plan: LogicalPlan, source: Seq[LogicalPlan]): Seq[LogicalPlan] = plan +: source override protected def build(plan: LogicalPlan, lhs: Seq[LogicalPlan], rhs: Seq[LogicalPlan]): Seq[LogicalPlan] = (plan +: lhs) ++ rhs -} \ No newline at end of file +} diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v2_3/Compatibility.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v2_3/Compatibility.scala index 4cb91688c4f62..423f7957ec803 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v2_3/Compatibility.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v2_3/Compatibility.scala @@ -19,6 +19,8 @@ */ package org.neo4j.cypher.internal.compatibility.v2_3 +import java.util.Collections.emptyList + import org.neo4j.cypher.internal._ import org.neo4j.cypher.internal.compatibility._ import org.neo4j.cypher.internal.compiler.v2_3.executionplan.{EntityAccessor, ExecutionPlan => ExecutionPlan_v2_3} @@ -31,7 +33,7 @@ import org.neo4j.cypher.internal.spi.v3_2.TransactionalContextWrapper import org.neo4j.graphdb.{Node, Relationship} import org.neo4j.kernel.GraphDatabaseQueryService import org.neo4j.kernel.api.ExecutingQuery.PlannerInfo -import org.neo4j.kernel.api.KernelAPI +import org.neo4j.kernel.api.{IndexUsage, KernelAPI} import org.neo4j.kernel.impl.core.NodeManager import org.neo4j.kernel.impl.query.QueryExecutionMonitor import org.neo4j.kernel.monitoring.{Monitors => KernelMonitors} @@ -114,7 +116,7 @@ trait Compatibility { def isStale(lastCommittedTxId: LastCommittedTxIdProvider, ctx: TransactionalContextWrapper): Boolean = inner.isStale(lastCommittedTxId, TransactionBoundGraphStatistics(ctx.readOperations)) - override def plannerInfo = new PlannerInfo(inner.plannerUsed.name, inner.runtimeUsed.name) + override def plannerInfo = new PlannerInfo(inner.plannerUsed.name, inner.runtimeUsed.name, emptyList[IndexUsage]) } } diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_1/Compatibility.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_1/Compatibility.scala index bc6307d2f74de..296752e2eb7c5 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_1/Compatibility.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_1/Compatibility.scala @@ -19,6 +19,8 @@ */ package org.neo4j.cypher.internal.compatibility.v3_1 +import java.util.Collections.emptyList + import org.neo4j.cypher.internal._ import org.neo4j.cypher.internal.compatibility._ import org.neo4j.cypher.internal.compatibility.v3_1.helpers._ @@ -31,7 +33,7 @@ import org.neo4j.cypher.internal.spi.v3_1.{TransactionalContextWrapper => Transa import org.neo4j.cypher.internal.spi.v3_2.{TransactionalContextWrapper => TransactionalContextWrapperV3_2} import org.neo4j.kernel.GraphDatabaseQueryService import org.neo4j.kernel.api.ExecutingQuery.PlannerInfo -import org.neo4j.kernel.api.KernelAPI +import org.neo4j.kernel.api.{IndexUsage, KernelAPI} import org.neo4j.kernel.impl.query.QueryExecutionMonitor import org.neo4j.kernel.monitoring.{Monitors => KernelMonitors} import org.neo4j.logging.Log @@ -115,7 +117,7 @@ trait Compatibility { def isStale(lastCommittedTxId: LastCommittedTxIdProvider, ctx: TransactionalContextWrapperV3_2): Boolean = inner.isStale(lastCommittedTxId, TransactionBoundGraphStatistics(ctx.readOperations)) - override def plannerInfo = new PlannerInfo(inner.plannerUsed.name, inner.runtimeUsed.name) + override def plannerInfo = new PlannerInfo(inner.plannerUsed.name, inner.runtimeUsed.name, emptyList[IndexUsage]) } } diff --git a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_2/Compatibility.scala b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_2/Compatibility.scala index cf38a7476d830..830a060108c51 100644 --- a/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_2/Compatibility.scala +++ b/community/cypher/cypher/src/main/scala/org/neo4j/cypher/internal/compatibility/v3_2/Compatibility.scala @@ -22,12 +22,13 @@ package org.neo4j.cypher.internal.compatibility.v3_2 import org.neo4j.cypher.internal._ import org.neo4j.cypher.internal.compatibility._ import org.neo4j.cypher.internal.compiler.v3_2 -import org.neo4j.cypher.internal.compiler.v3_2.executionplan.{ExecutionPlan => ExecutionPlan_v3_2} +import org.neo4j.cypher.internal.compiler.v3_2.executionplan.{LegacyNodeIndexUsage, LegacyRelationshipIndexUsage, SchemaIndexScanUsage, SchemaIndexSeekUsage, ExecutionPlan => ExecutionPlan_v3_2} import org.neo4j.cypher.internal.compiler.v3_2.tracing.rewriters.RewriterStepSequencer import org.neo4j.cypher.internal.compiler.v3_2.{InfoLogger, ExplainMode => ExplainModev3_2, NormalMode => NormalModev3_2, ProfileMode => ProfileModev3_2, _} import org.neo4j.cypher.internal.spi.v3_2.TransactionBoundQueryContext.IndexSearchMonitor import org.neo4j.cypher.internal.spi.v3_2._ import org.neo4j.kernel.api.ExecutingQuery.PlannerInfo +import org.neo4j.kernel.api.IndexUsage.{legacyIndexUsage, schemaIndexUsage} import org.neo4j.kernel.api.KernelAPI import org.neo4j.kernel.impl.query.QueryExecutionMonitor import org.neo4j.kernel.monitoring.{Monitors => KernelMonitors} @@ -108,7 +109,15 @@ trait Compatibility { def isStale(lastCommittedTxId: LastCommittedTxIdProvider, ctx: TransactionalContextWrapper): Boolean = inner.isStale(lastCommittedTxId, TransactionBoundGraphStatistics(ctx.readOperations)) - override def plannerInfo = new PlannerInfo(inner.plannerUsed.name, inner.runtimeUsed.name) + override def plannerInfo = { + import scala.collection.JavaConverters._ + new PlannerInfo(inner.plannerUsed.name, inner.runtimeUsed.name, inner.plannedIndexUsage.map { + case SchemaIndexSeekUsage(identifier, label, propertyKey) => schemaIndexUsage(identifier, label, propertyKey) + case SchemaIndexScanUsage(identifier, label, propertyKey) => schemaIndexUsage(identifier, label, propertyKey) + case LegacyNodeIndexUsage(identifier, index) => legacyIndexUsage(identifier, "NODE", index) + case LegacyRelationshipIndexUsage(identifier, index) => legacyIndexUsage(identifier, "RELATIONSHIP", index) + }.asJava) + } } } diff --git a/community/kernel/src/main/java/org/neo4j/helpers/ConcurrentTransfer.java b/community/kernel/src/main/java/org/neo4j/helpers/FailableConcurrentTransfer.java similarity index 70% rename from community/kernel/src/main/java/org/neo4j/helpers/ConcurrentTransfer.java rename to community/kernel/src/main/java/org/neo4j/helpers/FailableConcurrentTransfer.java index fab1f6c257304..c6c97f89678e8 100644 --- a/community/kernel/src/main/java/org/neo4j/helpers/ConcurrentTransfer.java +++ b/community/kernel/src/main/java/org/neo4j/helpers/FailableConcurrentTransfer.java @@ -20,17 +20,19 @@ package org.neo4j.helpers; import java.util.concurrent.CountDownLatch; -import java.util.function.Consumer; -import java.util.function.Supplier; + +import org.neo4j.function.FailableConsumer; +import org.neo4j.function.ThrowingSupplier; /** * Abstracts a meeting point between two threads, where a reference can change hands. It is essentially * a latch where a reference to a value can be set while a thread waits on it. */ -public class ConcurrentTransfer implements Consumer, Supplier +public class FailableConcurrentTransfer implements FailableConsumer, ThrowingSupplier { private final CountDownLatch latch = new CountDownLatch( 1 ); private TYPE value; + private Exception failure; @Override public void accept( TYPE value ) @@ -40,7 +42,14 @@ public void accept( TYPE value ) } @Override - public TYPE get() + public void fail( Exception failure ) + { + this.failure = failure; + latch.countDown(); + } + + @Override + public TYPE get() throws Exception { try { @@ -50,6 +59,16 @@ public TYPE get() { throw new RuntimeException( "Thread interrupted", e ); } + if ( failure != null ) + { + throw failure; + } return value; } + + @Override + public String toString() + { + return String.format( "ConcurrentTransfer{%s}", latch.getCount() == 1 ? "" : value ); + } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/api/ExecutingQuery.java b/community/kernel/src/main/java/org/neo4j/kernel/api/ExecutingQuery.java index aa6fdfcca8668..520c71399acbb 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/api/ExecutingQuery.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/api/ExecutingQuery.java @@ -19,6 +19,9 @@ */ package org.neo4j.kernel.api; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicLongFieldUpdater; @@ -31,6 +34,7 @@ import org.neo4j.time.CpuClock; import org.neo4j.time.SystemNanoClock; +import static java.util.Collections.emptyList; import static java.util.concurrent.TimeUnit.NANOSECONDS; import static java.util.concurrent.atomic.AtomicLongFieldUpdater.newUpdater; @@ -42,11 +46,12 @@ public class ExecutingQuery public static class QueryInfo { public final String text; - public final Map parameters; + public final Map parameters; public final String planner; public final String runtime; + private final List indexes; - private QueryInfo( String text, Map parameters, PlannerInfo plannerInfo ) + private QueryInfo( String text, Map parameters, PlannerInfo plannerInfo ) { this.text = text; this.parameters = parameters; @@ -54,24 +59,38 @@ private QueryInfo( String text, Map parameters, PlannerInfo plann { this.planner = plannerInfo.planner; this.runtime = plannerInfo.runtime; + this.indexes = plannerInfo.indexes; } else { this.planner = null; this.runtime = null; + this.indexes = emptyList(); } } + + public List> indexes() + { + List> used = new ArrayList<>( this.indexes.size() ); + for ( IndexUsage index : indexes ) + { + used.add( index.asMap() ); + } + return used; + } } public static class PlannerInfo { - private final String planner; - private final String runtime; + final String planner; + final String runtime; + final List indexes; - public PlannerInfo( String planner, String runtime ) + public PlannerInfo( String planner, String runtime, List indexes ) { this.planner = planner; this.runtime = runtime; + this.indexes = indexes; } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/api/IndexUsage.java b/community/kernel/src/main/java/org/neo4j/kernel/api/IndexUsage.java new file mode 100644 index 0000000000000..249e321148588 --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/api/IndexUsage.java @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.api; + +import java.util.HashMap; +import java.util.Map; + +public abstract class IndexUsage +{ + public static IndexUsage schemaIndexUsage( String identifier, String label, String propertyKey ) + { + return new SchemaIndexUsage( identifier, label, propertyKey ); + } + + public static IndexUsage legacyIndexUsage( String identifier, String entityType, String index ) + { + return new LegacyIndexUsage( identifier, index, entityType ); + } + + abstract Map asMap(); + + final String identifier; + + private IndexUsage( String identifier ) + { + this.identifier = identifier; + } + + private static class SchemaIndexUsage extends IndexUsage + { + private final String label; + private final String propertyKey; + + private SchemaIndexUsage( String identifier, String label, String propertyKey ) + { + super( identifier ); + this.label = label; + this.propertyKey = propertyKey; + } + + Map asMap() + { + Map map = new HashMap<>(); + map.put( "indexType", "SCHEMA INDEX" ); + map.put( "entityType", "NODE" ); + map.put( "identifier", identifier ); + map.put( "label", label ); + map.put( "propertyKey", propertyKey ); + return map; + } + } + + private static class LegacyIndexUsage extends IndexUsage + { + private final String index; + private final String entityType; + + private LegacyIndexUsage( String identifier, String index, String entityType ) + { + super( identifier ); + this.index = index; + this.entityType = entityType; + } + + @Override + Map asMap() + { + Map map = new HashMap<>(); + map.put( "indexType", "LEGACY INDEX" ); + map.put( "entityType", entityType ); + map.put( "identifier", identifier ); + map.put( "indexName", index ); + return map; + } + } +} diff --git a/community/kernel/src/test/java/org/neo4j/kernel/api/ExecutingQueryTest.java b/community/kernel/src/test/java/org/neo4j/kernel/api/ExecutingQueryTest.java index 92508b35d6ed6..e78557906846d 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/api/ExecutingQueryTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/api/ExecutingQueryTest.java @@ -39,6 +39,7 @@ import org.neo4j.time.Clocks; import org.neo4j.time.FakeClock; +import static java.util.Collections.emptyList; import static java.util.Collections.singletonMap; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.hasEntry; @@ -78,7 +79,7 @@ public void shouldTransitionBetweenStates() throws Exception assertThat( query.status(), hasEntry( "state", "PLANNING" ) ); // when - query.planningCompleted( new ExecutingQuery.PlannerInfo( "the-planner", "the-runtime" ) ); + query.planningCompleted( new ExecutingQuery.PlannerInfo( "the-planner", "the-runtime", emptyList() ) ); // then assertThat( query.status(), hasEntry( "state", "RUNNING" ) ); @@ -104,7 +105,7 @@ public void shouldReportPlanningTime() throws Exception // when clock.forward( 16, TimeUnit.MILLISECONDS ); - query.planningCompleted( new ExecutingQuery.PlannerInfo( "the-planner", "the-runtime" ) ); + query.planningCompleted( new ExecutingQuery.PlannerInfo( "the-planner", "the-runtime", emptyList() ) ); clock.forward( 200, TimeUnit.MILLISECONDS ); // then @@ -116,7 +117,7 @@ public void shouldReportPlanningTime() throws Exception public void shouldReportWaitTime() throws Exception { // given - query.planningCompleted( new ExecutingQuery.PlannerInfo( "the-planner", "the-runtime" ) ); + query.planningCompleted( new ExecutingQuery.PlannerInfo( "the-planner", "the-runtime", emptyList() ) ); // then assertEquals( singletonMap( "state", "RUNNING" ), query.status() ); diff --git a/community/kernel/src/test/java/org/neo4j/test/rule/concurrent/ThreadingRule.java b/community/kernel/src/test/java/org/neo4j/test/rule/concurrent/ThreadingRule.java index 4a46e07989ba8..5e91a61a9fbae 100644 --- a/community/kernel/src/test/java/org/neo4j/test/rule/concurrent/ThreadingRule.java +++ b/community/kernel/src/test/java/org/neo4j/test/rule/concurrent/ThreadingRule.java @@ -22,6 +22,7 @@ import org.junit.rules.ExternalResource; import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -30,12 +31,16 @@ import java.util.function.Consumer; import java.util.function.Predicate; +import org.neo4j.function.FailableConsumer; import org.neo4j.function.Predicates; import org.neo4j.function.ThrowingFunction; -import org.neo4j.helpers.ConcurrentTransfer; +import org.neo4j.function.ThrowingPredicate; +import org.neo4j.helpers.FailableConcurrentTransfer; import org.neo4j.test.Barrier; import org.neo4j.test.ReflectionUtil; +import static org.neo4j.function.ThrowingPredicate.throwingPredicate; + public class ThreadingRule extends ExternalResource { private ExecutorService executor; @@ -66,22 +71,40 @@ protected void after() public Future execute( ThrowingFunction function, FROM parameter ) { - return executor.submit( task( Barrier.NONE, function, parameter, (t) -> {} ) ); + return executor.submit( task( Barrier.NONE, function, parameter, new FailableConsumer() + { + @Override + public void fail( Exception failure ) + { + } + + @Override + public void accept( Thread thread ) + { + } + } ) ); } public Future executeAndAwait( ThrowingFunction function, FROM parameter, Predicate threadCondition, - long timeout, TimeUnit unit ) throws TimeoutException, InterruptedException + long timeout, TimeUnit unit ) throws TimeoutException, InterruptedException, ExecutionException { - ConcurrentTransfer threadTransfer = new ConcurrentTransfer<>(); + FailableConcurrentTransfer threadTransfer = new FailableConcurrentTransfer<>(); Future future = executor.submit( task( Barrier.NONE, function, parameter, threadTransfer ) ); - Predicates.await( threadTransfer, threadCondition, timeout, unit ); + try + { + Predicates.awaitEx( threadTransfer, throwingPredicate( threadCondition ), timeout, unit ); + } + catch ( Exception e ) + { + throw new ExecutionException( e ); + } return future; } private static Callable task( final Barrier barrier, final ThrowingFunction function, final FROM parameter, - final Consumer threadConsumer ) + final FailableConsumer threadConsumer ) { return () -> { Thread thread = Thread.currentThread(); @@ -93,6 +116,11 @@ private static Callable task( { return function.apply( parameter ); } + catch ( Exception failure ) + { + threadConsumer.fail( failure ); + throw failure; + } finally { thread.setName( name ); diff --git a/enterprise/kernel/src/main/java/org/neo4j/kernel/enterprise/builtinprocs/QueryStatusResult.java b/enterprise/kernel/src/main/java/org/neo4j/kernel/enterprise/builtinprocs/QueryStatusResult.java index 8762ae4118f94..2f42bbf4fab77 100644 --- a/enterprise/kernel/src/main/java/org/neo4j/kernel/enterprise/builtinprocs/QueryStatusResult.java +++ b/enterprise/kernel/src/main/java/org/neo4j/kernel/enterprise/builtinprocs/QueryStatusResult.java @@ -22,6 +22,7 @@ import java.time.Instant; import java.time.OffsetDateTime; import java.time.ZoneId; +import java.util.List; import java.util.Map; import org.neo4j.kernel.api.ExecutingQuery; @@ -63,6 +64,7 @@ public class QueryStatusResult /** EXPERIMENTAL: added in Neo4j 3.2 */ public final long waitTimeMillis; // TODO: we want this field to be of a Duration type (when Cypher supports that) public final Map metaData; + public final List> indexes; QueryStatusResult( ExecutingQuery q ) throws InvalidArgumentsException { @@ -108,6 +110,7 @@ private QueryStatusResult( this.waitTimeMillis = waitTimeMillis; this.planner = query.planner; this.runtime = query.runtime; + this.indexes = query.indexes(); } private static String formatTime( final long startTime ) diff --git a/enterprise/kernel/src/test/java/org/neo4j/kernel/enterprise/builtinprocs/ListQueriesProcedureTest.java b/enterprise/kernel/src/test/java/org/neo4j/kernel/enterprise/builtinprocs/ListQueriesProcedureTest.java index bfb5abea6c47a..5c2f799e75476 100644 --- a/enterprise/kernel/src/test/java/org/neo4j/kernel/enterprise/builtinprocs/ListQueriesProcedureTest.java +++ b/enterprise/kernel/src/test/java/org/neo4j/kernel/enterprise/builtinprocs/ListQueriesProcedureTest.java @@ -19,8 +19,10 @@ */ package org.neo4j.kernel.enterprise.builtinprocs; +import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; import java.util.function.BiConsumer; import java.util.function.Supplier; @@ -31,6 +33,7 @@ import org.neo4j.graphdb.Node; import org.neo4j.graphdb.Result; import org.neo4j.graphdb.Transaction; +import org.neo4j.graphdb.factory.GraphDatabaseBuilder; import org.neo4j.kernel.enterprise.ImpermanentEnterpriseDatabaseRule; import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; import org.neo4j.test.rule.DatabaseRule; @@ -38,19 +41,30 @@ import static java.util.concurrent.TimeUnit.SECONDS; import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.hasKey; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; +import static org.neo4j.graphdb.Label.label; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.cypher_hints_error; import static org.neo4j.test.rule.concurrent.ThreadingRule.waitingWhileIn; public class ListQueriesProcedureTest { @Rule - public final DatabaseRule db = new ImpermanentEnterpriseDatabaseRule(); + public final DatabaseRule db = new ImpermanentEnterpriseDatabaseRule() + { + @Override + protected void configure( GraphDatabaseBuilder builder ) + { + builder.setConfig( cypher_hints_error, "true" ); + } + }; @Rule public final ThreadingRule threads = new ThreadingRule(); @@ -139,6 +153,122 @@ public void shouldContainSpecificConnectionDetails() throws Exception assertThat( data, hasKey( "requestUri" ) ); } + @Test + public void shouldListUsedIndexes() throws Exception + { + // given + String label = "IndexedLabel", property = "indexedProperty"; + try ( Transaction tx = db.beginTx() ) + { + db.schema().indexFor( label( label ) ).on( property ).create(); + tx.success(); + } + shouldListUsedIndexes( label, property ); + } + + @Test + public void shouldListUsedUniqueIndexes() throws Exception + { + // given + String label = "UniqueLabel", property = "uniqueProperty"; + try ( Transaction tx = db.beginTx() ) + { + db.schema().constraintFor( label( label ) ).assertPropertyIsUnique( property ).create(); + tx.success(); + } + shouldListUsedIndexes( label, property ); + } + + @Test + public void shouldListIndexesUsedForScans() throws Exception + { + // given + String QUERY = "MATCH (n:Node) USING INDEX n:Node(value) WHERE 1 < n.value < 10 SET n.value = 2"; + try ( Transaction tx = db.beginTx() ) + { + db.schema().indexFor( label( "Node" ) ).on( "value" ).create(); + tx.success(); + } + try ( Resource test = test( () -> + { + Node node = db.createNode( label( "Node" ) ); + node.setProperty( "value", 5L ); + return node; + }, Transaction::acquireWriteLock, QUERY ) ) + { + // when + Map data = getQueryListing( QUERY ); + + // then + assertThat( data, hasEntry( equalTo( "indexes" ), instanceOf( List.class ) ) ); + @SuppressWarnings( "unchecked" ) + List> indexes = (List>) data.get( "indexes" ); + assertEquals( "number of indexes used", 1, indexes.size() ); + Map index = indexes.get( 0 ); + assertThat( index, hasEntry( "identifier", "n" ) ); + assertThat( index, hasEntry( "label", "Node" ) ); + assertThat( index, hasEntry( "propertyKey", "value" ) ); + } + } + + private void shouldListUsedIndexes( String label, String property ) throws Exception + { + // given + String QUERY1 = "MATCH (n:" + label + "{" + property + ":5}) USING INDEX n:" + label + "(" + property + + ") SET n." + property + " = 3"; + try ( Resource test = test( () -> + { + Node node = db.createNode( label( label ) ); + node.setProperty( property, 5L ); + return node; + }, Transaction::acquireWriteLock, QUERY1 ) ) + { + // when + Map data = getQueryListing( QUERY1 ); + + // then + assertThat( data, hasEntry( equalTo( "indexes" ), instanceOf( List.class ) ) ); + @SuppressWarnings( "unchecked" ) + List> indexes = (List>) data.get( "indexes" ); + assertEquals( "number of indexes used", 1, indexes.size() ); + Map index = indexes.get( 0 ); + assertThat( index, hasEntry( "identifier", "n" ) ); + assertThat( index, hasEntry( "label", label ) ); + assertThat( index, hasEntry( "propertyKey", property ) ); + } + + // given + String QUERY2 = "MATCH (n:" + label + "{" + property + ":3}) USING INDEX n:" + label + "(" + property + + ") MATCH (u:" + label + "{" + property + ":4}) USING INDEX u:" + label + "(" + property + + ") CREATE (n)-[:KNOWS]->(u)"; + try ( Resource test = test( () -> + { + Node node = db.createNode( label( label ) ); + node.setProperty( property, 4L ); + return node; + }, Transaction::acquireWriteLock, QUERY2 ) ) + { + // when + Map data = getQueryListing( QUERY2 ); + + // then + assertThat( data, hasEntry( equalTo( "indexes" ), instanceOf( List.class ) ) ); + @SuppressWarnings( "unchecked" ) + List> indexes = (List>) data.get( "indexes" ); + assertEquals( "number of indexes used", 2, indexes.size() ); + + Map index1 = indexes.get( 0 ); + assertThat( index1, hasEntry( "identifier", "n" ) ); + assertThat( index1, hasEntry( "label", label ) ); + assertThat( index1, hasEntry( "propertyKey", property ) ); + + Map index2 = indexes.get( 1 ); + assertThat( index2, hasEntry( "identifier", "u" ) ); + assertThat( index2, hasEntry( "label", label ) ); + assertThat( index2, hasEntry( "propertyKey", property ) ); + } + } + private Map getQueryListing( String query ) { try ( Result rows = db.execute( "CALL dbms.listQueries" ) ) @@ -179,7 +309,7 @@ public T resource() } private Resource test( Supplier setup, BiConsumer lock, String query ) - throws TimeoutException, InterruptedException + throws TimeoutException, InterruptedException, ExecutionException { CountDownLatch resourceLocked = new CountDownLatch( 1 ), listQueriesLatch = new CountDownLatch( 1 ); T resource;