From a24e5f2f17d7a513514135fe3b3f43554e83a0f9 Mon Sep 17 00:00:00 2001 From: Tobias Lindaaker Date: Thu, 23 Apr 2015 17:02:06 +0200 Subject: [PATCH] Outline of using tracers for profiling --- .../internal/compiler/v2_3/birk/Javac.java | 9 +- .../v2_3/birk/QueryExecutionTracer.java | 36 ++++ .../v2_3/birk/profiling/ProfilingTracer.java | 167 ++++++++++++++++++ .../compiler/v2_3/birk/CodeGenerator.scala | 14 +- .../v2_3/birk/QueryExecutionEvent.java | 50 ++++++ .../CompiledExecutionResult.scala | 5 +- .../executionplan/ExecutionPlanBuilder.scala | 36 +++- .../InternalPlanDescription.scala | 1 + .../PlanDescriptionArgumentSerializer.scala | 1 + .../v2_3/birk/CodeGeneratorTest.scala | 3 +- .../compiler/v2_3/birk/il/CodeGenSugar.scala | 11 +- .../birk/profiling/ProfilingTracerTest.scala | 85 +++++++++ .../neo4j/cypher/ProfilerAcceptanceTest.scala | 18 +- .../internal/RewindableExecutionResult.scala | 1 + .../src/test/scala/org/neo4j/cypher/tempgen | 148 ++++++++++++++++ 15 files changed, 560 insertions(+), 25 deletions(-) create mode 100644 community/cypher/cypher-compiler-2.3/src/main/java/org/neo4j/cypher/internal/compiler/v2_3/birk/QueryExecutionTracer.java create mode 100644 community/cypher/cypher-compiler-2.3/src/main/java/org/neo4j/cypher/internal/compiler/v2_3/birk/profiling/ProfilingTracer.java create mode 100644 community/cypher/cypher-compiler-2.3/src/main/scala/org/neo4j/cypher/internal/compiler/v2_3/birk/QueryExecutionEvent.java create mode 100644 community/cypher/cypher-compiler-2.3/src/test/scala/org/neo4j/cypher/internal/compiler/v2_3/birk/profiling/ProfilingTracerTest.scala create mode 100644 community/cypher/cypher/src/test/scala/org/neo4j/cypher/tempgen diff --git a/community/cypher/cypher-compiler-2.3/src/main/java/org/neo4j/cypher/internal/compiler/v2_3/birk/Javac.java b/community/cypher/cypher-compiler-2.3/src/main/java/org/neo4j/cypher/internal/compiler/v2_3/birk/Javac.java index e69d66797be59..648d14a1f9c93 100644 --- a/community/cypher/cypher-compiler-2.3/src/main/java/org/neo4j/cypher/internal/compiler/v2_3/birk/Javac.java +++ b/community/cypher/cypher-compiler-2.3/src/main/java/org/neo4j/cypher/internal/compiler/v2_3/birk/Javac.java @@ -40,13 +40,12 @@ import javax.tools.SimpleJavaFileObject; import javax.tools.StandardLocation; import javax.tools.ToolProvider; - - import org.neo4j.cypher.internal.compiler.v2_3.ExecutionMode; import org.neo4j.cypher.internal.compiler.v2_3.TaskCloser; import org.neo4j.cypher.internal.compiler.v2_3.executionplan.InternalExecutionResult; import org.neo4j.cypher.internal.compiler.v2_3.planDescription.InternalPlanDescription; import org.neo4j.cypher.internal.compiler.v2_3.planner.CantCompileQueryException; +import org.neo4j.function.Supplier; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.kernel.api.Statement; @@ -99,12 +98,12 @@ public static Class compile( String className, String c } public static InternalExecutionResult newInstance( Class clazz, TaskCloser closer, Statement statement, - GraphDatabaseService db, ExecutionMode executionMode, InternalPlanDescription description, Map params) + GraphDatabaseService db, ExecutionMode executionMode, Supplier description, QueryExecutionTracer tracer, Map params) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException { Constructor constructor = - clazz.getDeclaredConstructor( TaskCloser.class, Statement.class, GraphDatabaseService.class, ExecutionMode.class, InternalPlanDescription.class , Map.class); - return constructor.newInstance( closer, statement, db, executionMode, description, params ); + clazz.getDeclaredConstructor( TaskCloser.class, Statement.class, GraphDatabaseService.class, ExecutionMode.class, Supplier.class, QueryExecutionTracer.class, Map.class); + return constructor.newInstance( closer, statement, db, executionMode, description, tracer, params ); } private static class InMemSource extends SimpleJavaFileObject diff --git a/community/cypher/cypher-compiler-2.3/src/main/java/org/neo4j/cypher/internal/compiler/v2_3/birk/QueryExecutionTracer.java b/community/cypher/cypher-compiler-2.3/src/main/java/org/neo4j/cypher/internal/compiler/v2_3/birk/QueryExecutionTracer.java new file mode 100644 index 0000000000000..5aa290d2b1919 --- /dev/null +++ b/community/cypher/cypher-compiler-2.3/src/main/java/org/neo4j/cypher/internal/compiler/v2_3/birk/QueryExecutionTracer.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.cypher.internal.compiler.v2_3.birk; + +import org.neo4j.cypher.internal.compiler.v2_3.planDescription.Id; + +public interface QueryExecutionTracer +{ + QueryExecutionEvent executeQuery( Id queryId ); + + QueryExecutionTracer NONE = new QueryExecutionTracer() + { + @Override + public QueryExecutionEvent executeQuery( Id queryId ) + { + return QueryExecutionEvent.NONE; + } + }; +} diff --git a/community/cypher/cypher-compiler-2.3/src/main/java/org/neo4j/cypher/internal/compiler/v2_3/birk/profiling/ProfilingTracer.java b/community/cypher/cypher-compiler-2.3/src/main/java/org/neo4j/cypher/internal/compiler/v2_3/birk/profiling/ProfilingTracer.java new file mode 100644 index 0000000000000..da9315f4f28fe --- /dev/null +++ b/community/cypher/cypher-compiler-2.3/src/main/java/org/neo4j/cypher/internal/compiler/v2_3/birk/profiling/ProfilingTracer.java @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.neo4j.cypher.internal.compiler.v2_3.birk.profiling; + +import java.util.HashMap; +import java.util.Map; + +import org.neo4j.cypher.internal.compiler.v2_3.birk.QueryExecutionEvent; +import org.neo4j.cypher.internal.compiler.v2_3.birk.QueryExecutionTracer; +import org.neo4j.cypher.internal.compiler.v2_3.planDescription.Id; + +public class ProfilingTracer implements QueryExecutionTracer +{ + public interface ProfilingInformation + { + long time(); + long dbHits(); + long rows(); + } + + public interface Clock + { + long nanoTime(); + + Clock SYSTEM_TIMER = new Clock() + { + @Override + public long nanoTime() + { + return System.nanoTime(); + } + }; + } + + private static final Data ZERO = new Data(); + + private final Clock clock; + private final Map data = new HashMap<>(); + + public ProfilingTracer() + { + this( Clock.SYSTEM_TIMER ); + } + + ProfilingTracer( Clock clock ) + { + this.clock = clock; + } + + public ProfilingInformation get( Id query ) + { + Data value = data.get( query ); + return value == null ? ZERO : value; + } + + public long timeOf( Id query ) + { + return get( query ).time(); + } + + public long dbHitsOf( Id query ) + { + return get( query ).dbHits(); + } + + public long rowsOf( Id query ) + { + return get( query ).rows(); + } + + @Override + public QueryExecutionEvent executeQuery( Id queryId ) + { + Data data = this.data.get( queryId ); + if ( data == null && queryId != null ) + { + this.data.put( queryId, data = new Data() ); + } + return new ExecutionEvent( clock, data ); + } + + private static class ExecutionEvent implements QueryExecutionEvent + { + private final long start; + private final Clock clock; + private final Data data; + private long hitCount; + private long rowCount; + + public ExecutionEvent( Clock clock, Data data ) + { + this.clock = clock; + this.data = data; + this.start = clock.nanoTime(); + } + + @Override + public void close() + { + long executionTime = clock.nanoTime() - start; + if ( data != null ) + { + data.update( executionTime, hitCount, rowCount ); + } + } + + @Override + public void dbHit() + { + hitCount++; + } + + @Override + public void row() + { + rowCount++; + } + } + + private static class Data implements ProfilingInformation + { + private long time, hits, rows; + + public void update( long time, long hits, long rows ) + { + this.time += time; + this.hits += hits; + this.rows += rows; + } + + @Override + public long time() + { + return time; + } + + @Override + public long dbHits() + { + return hits; + } + + @Override + public long rows() + { + return rows; + } + } +} diff --git a/community/cypher/cypher-compiler-2.3/src/main/scala/org/neo4j/cypher/internal/compiler/v2_3/birk/CodeGenerator.scala b/community/cypher/cypher-compiler-2.3/src/main/scala/org/neo4j/cypher/internal/compiler/v2_3/birk/CodeGenerator.scala index 437be0f54036d..d65aeee14cb1c 100644 --- a/community/cypher/cypher-compiler-2.3/src/main/scala/org/neo4j/cypher/internal/compiler/v2_3/birk/CodeGenerator.scala +++ b/community/cypher/cypher-compiler-2.3/src/main/scala/org/neo4j/cypher/internal/compiler/v2_3/birk/CodeGenerator.scala @@ -35,6 +35,7 @@ import org.neo4j.cypher.internal.compiler.v2_3.planner.logical.{LogicalPlan2Plan import org.neo4j.cypher.internal.compiler.v2_3.planner.{CantCompileQueryException, SemanticTable} import org.neo4j.cypher.internal.compiler.v2_3.spi.{InstrumentedGraphStatistics, PlanContext} import org.neo4j.cypher.internal.compiler.v2_3.{ExecutionMode, GreedyPlannerName} +import org.neo4j.function.Supplier import org.neo4j.graphdb.GraphDatabaseService import org.neo4j.helpers.Clock import org.neo4j.kernel.api.Statement @@ -114,6 +115,7 @@ object CodeGenerator { s"""package $packageName; | |import org.neo4j.helpers.collection.Visitor; + |import org.neo4j.function.Supplier; |import org.neo4j.graphdb.GraphDatabaseService; |import org.neo4j.kernel.api.Statement; |import org.neo4j.kernel.api.exceptions.KernelException; @@ -128,6 +130,8 @@ object CodeGenerator { |import org.neo4j.cypher.internal.compiler.v2_3.planDescription.InternalPlanDescription; |import org.neo4j.cypher.internal.compiler.v2_3.ExecutionMode; |import org.neo4j.cypher.internal.compiler.v2_3.TaskCloser; + |import org.neo4j.cypher.internal.compiler.v2_3.birk.QueryExecutionTracer; + |import org.neo4j.cypher.internal.compiler.v2_3.birk.QueryExecutionEvent; |import java.util.Map; | |$imports @@ -137,12 +141,14 @@ object CodeGenerator { |private final ReadOperations ro; |private final GraphDatabaseService db; |private final Map params; + |private final QueryExecutionTracer tracer; | - |public $className( TaskCloser closer, Statement statement, GraphDatabaseService db, ExecutionMode executionMode, InternalPlanDescription description, Map params ) + |public $className( TaskCloser closer, Statement statement, GraphDatabaseService db, ExecutionMode executionMode, Supplier description, QueryExecutionTracer tracer, Map params ) |{ | super( closer, statement, executionMode, description ); | this.ro = statement.readOperations(); | this.db = db; + | this.tracer = tracer; | this.params = params; |} | @@ -199,8 +205,10 @@ class CodeGenerator { val idMap = LogicalPlanIdentificationBuilder(plan) val description: InternalPlanDescription = LogicalPlan2PlanDescription(plan, idMap) - val builder = (st: Statement, db: GraphDatabaseService, mode: ExecutionMode, params: Map[String, Any], closer: TaskCloser) => - Javac.newInstance(clazz, closer, st, db, mode, description, asJavaHashMap(params)) + val builder = (st: Statement, db: GraphDatabaseService, mode: ExecutionMode, tracing:(InternalPlanDescription=>(Supplier[InternalPlanDescription],Option[QueryExecutionTracer])), params: Map[String, Any], closer: TaskCloser) => { + val (supplier, tracer) = tracing(description) + Javac.newInstance(clazz, closer, st, db, mode, supplier, tracer.getOrElse(QueryExecutionTracer.NONE), asJavaHashMap(params)) + } val columns = res.nodes ++ res.relationships ++ res.other CompiledPlan(updating = false, None, fp, GreedyPlannerName, description, columns, builder) diff --git a/community/cypher/cypher-compiler-2.3/src/main/scala/org/neo4j/cypher/internal/compiler/v2_3/birk/QueryExecutionEvent.java b/community/cypher/cypher-compiler-2.3/src/main/scala/org/neo4j/cypher/internal/compiler/v2_3/birk/QueryExecutionEvent.java new file mode 100644 index 0000000000000..9ee7fe7996b4b --- /dev/null +++ b/community/cypher/cypher-compiler-2.3/src/main/scala/org/neo4j/cypher/internal/compiler/v2_3/birk/QueryExecutionEvent.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.neo4j.cypher.internal.compiler.v2_3.birk; + +public interface QueryExecutionEvent extends AutoCloseable +{ + void dbHit(); + + void row(); + + @Override + void close(); + + QueryExecutionEvent NONE = new QueryExecutionEvent() + { + @Override + public void dbHit() + { + } + + @Override + public void row() + { + throw new UnsupportedOperationException( "not implemented" ); + } + + @Override + public void close() + { + } + }; +} diff --git a/community/cypher/cypher-compiler-2.3/src/main/scala/org/neo4j/cypher/internal/compiler/v2_3/executionplan/CompiledExecutionResult.scala b/community/cypher/cypher-compiler-2.3/src/main/scala/org/neo4j/cypher/internal/compiler/v2_3/executionplan/CompiledExecutionResult.scala index d446b7ebb1c46..86ee1843f42c8 100644 --- a/community/cypher/cypher-compiler-2.3/src/main/scala/org/neo4j/cypher/internal/compiler/v2_3/executionplan/CompiledExecutionResult.scala +++ b/community/cypher/cypher-compiler-2.3/src/main/scala/org/neo4j/cypher/internal/compiler/v2_3/executionplan/CompiledExecutionResult.scala @@ -29,6 +29,7 @@ import org.neo4j.cypher.internal.compiler.v2_3.notification.InternalNotification import org.neo4j.cypher.internal.compiler.v2_3.planDescription.InternalPlanDescription import org.neo4j.cypher.internal.compiler.v2_3.planDescription.InternalPlanDescription.Arguments.{Runtime, Planner} import org.neo4j.cypher.internal.compiler.v2_3.{ExecutionMode, ExplainMode, ProfileMode, _} +import org.neo4j.function.Supplier import org.neo4j.graphdb.QueryExecutionType._ import org.neo4j.graphdb.Result.{ResultRow, ResultVisitor} import org.neo4j.graphdb._ @@ -40,7 +41,7 @@ import scala.collection.{Map, mutable} * Base class for compiled execution results, implements everything in InternalExecutionResult * except `javaColumns` and `accept` which should be implemented by the generated classes. */ -abstract class CompiledExecutionResult(taskCloser: TaskCloser, statement:Statement, executionMode:ExecutionMode, description:InternalPlanDescription) extends InternalExecutionResult { +abstract class CompiledExecutionResult(taskCloser: TaskCloser, statement:Statement, executionMode:ExecutionMode, description:Supplier[InternalPlanDescription]) extends InternalExecutionResult { self => import scala.collection.JavaConverters._ @@ -118,7 +119,7 @@ abstract class CompiledExecutionResult(taskCloser: TaskCloser, statement:Stateme taskCloser.close(success = successful) } - override def executionPlanDescription() = description + override def executionPlanDescription() = description.get() def mode = executionMode diff --git a/community/cypher/cypher-compiler-2.3/src/main/scala/org/neo4j/cypher/internal/compiler/v2_3/executionplan/ExecutionPlanBuilder.scala b/community/cypher/cypher-compiler-2.3/src/main/scala/org/neo4j/cypher/internal/compiler/v2_3/executionplan/ExecutionPlanBuilder.scala index 5b4877ecbc678..64962a52dbc1e 100644 --- a/community/cypher/cypher-compiler-2.3/src/main/scala/org/neo4j/cypher/internal/compiler/v2_3/executionplan/ExecutionPlanBuilder.scala +++ b/community/cypher/cypher-compiler-2.3/src/main/scala/org/neo4j/cypher/internal/compiler/v2_3/executionplan/ExecutionPlanBuilder.scala @@ -19,19 +19,24 @@ */ package org.neo4j.cypher.internal.compiler.v2_3.executionplan -import org.neo4j.cypher.internal.compiler.v2_3._ import org.neo4j.cypher.internal.compiler.v2_3.ast.Statement +import org.neo4j.cypher.internal.compiler.v2_3.birk.QueryExecutionTracer +import org.neo4j.cypher.internal.compiler.v2_3.birk.profiling.ProfilingTracer import org.neo4j.cypher.internal.compiler.v2_3.commands._ +import org.neo4j.cypher.internal.compiler.v2_3.executionplan.ExecutionPlanBuilder.tracer import org.neo4j.cypher.internal.compiler.v2_3.executionplan.builders._ import org.neo4j.cypher.internal.compiler.v2_3.notification.InternalNotification import org.neo4j.cypher.internal.compiler.v2_3.pipes._ import org.neo4j.cypher.internal.compiler.v2_3.planDescription.InternalPlanDescription +import org.neo4j.cypher.internal.compiler.v2_3.planDescription.InternalPlanDescription.Arguments import org.neo4j.cypher.internal.compiler.v2_3.planner.logical.plans.LogicalPlan import org.neo4j.cypher.internal.compiler.v2_3.planner.{CantCompileQueryException, CantHandleQueryException} import org.neo4j.cypher.internal.compiler.v2_3.profiler.Profiler import org.neo4j.cypher.internal.compiler.v2_3.spi._ import org.neo4j.cypher.internal.compiler.v2_3.symbols.SymbolTable -import org.neo4j.cypher.internal.compiler.v2_3.{ExecutionMode, ProfileMode} +import org.neo4j.cypher.internal.compiler.v2_3.{ExecutionMode, ProfileMode, _} +import org.neo4j.function.Supplier +import org.neo4j.function.Suppliers.singleton import org.neo4j.graphdb.GraphDatabaseService import org.neo4j.graphdb.QueryExecutionType.QueryType import org.neo4j.helpers.Clock @@ -44,7 +49,12 @@ case class CompiledPlan(updating: Boolean, plannerUsed: PlannerName, planDescription: InternalPlanDescription, columns: Seq[String], - executionResultBuilder: (KernelStatement, GraphDatabaseService, ExecutionMode, Map[String, Any], TaskCloser) => InternalExecutionResult) + executionResultBuilder: (KernelStatement, + GraphDatabaseService, + ExecutionMode, + (InternalPlanDescription => (Supplier[InternalPlanDescription], Option[QueryExecutionTracer])), + Map[String, Any], + TaskCloser) => InternalExecutionResult ) case class PipeInfo(pipe: Pipe, updating: Boolean, @@ -95,7 +105,7 @@ class ExecutionPlanBuilder(graph: GraphDatabaseService, statsDivergenceThreshold new ExplainExecutionResult(taskCloser, compiledPlan.columns.toList, compiledPlan.planDescription, QueryType.READ_ONLY, inputQuery.notificationLogger.notifications) } else - compiledPlan.executionResultBuilder(kernelStatement, graph, executionMode, params, taskCloser) + compiledPlan.executionResultBuilder(kernelStatement, graph, executionMode, tracer(executionMode), params, taskCloser) } catch { case (t: Throwable) => taskCloser.close(success = false) @@ -195,3 +205,21 @@ class ExecutionPlanBuilder(graph: GraphDatabaseService, statsDivergenceThreshold builder.build(graph, queryId, planType, params, notificationLogger) } } + +object ExecutionPlanBuilder { + def tracer( mode: ExecutionMode ) = mode match { + case ProfileMode => + val tracer = new ProfilingTracer() + ( description: InternalPlanDescription ) => (new Supplier[InternalPlanDescription] { + override def get() = description map { + plan: InternalPlanDescription => + val data = tracer.get(plan.id) + plan. + addArgument( Arguments.DbHits( data.dbHits() ) ). + addArgument( Arguments.Rows( data.rows() ) ). + addArgument( Arguments.Time( data.time() ) ) + } + }, Some(tracer)) + case _ => (description: InternalPlanDescription) => (singleton(description), None) + } +} diff --git a/community/cypher/cypher-compiler-2.3/src/main/scala/org/neo4j/cypher/internal/compiler/v2_3/planDescription/InternalPlanDescription.scala b/community/cypher/cypher-compiler-2.3/src/main/scala/org/neo4j/cypher/internal/compiler/v2_3/planDescription/InternalPlanDescription.scala index 0ad60e28c2af5..3cefd10c4f183 100644 --- a/community/cypher/cypher-compiler-2.3/src/main/scala/org/neo4j/cypher/internal/compiler/v2_3/planDescription/InternalPlanDescription.scala +++ b/community/cypher/cypher-compiler-2.3/src/main/scala/org/neo4j/cypher/internal/compiler/v2_3/planDescription/InternalPlanDescription.scala @@ -74,6 +74,7 @@ sealed abstract class Argument extends Product { object InternalPlanDescription { object Arguments { + case class Time(value: Long) extends Argument case class Rows(value: Long) extends Argument case class DbHits(value: Long) extends Argument case class ColumnsLeft(value: Seq[String]) extends Argument diff --git a/community/cypher/cypher-compiler-2.3/src/main/scala/org/neo4j/cypher/internal/compiler/v2_3/planDescription/PlanDescriptionArgumentSerializer.scala b/community/cypher/cypher-compiler-2.3/src/main/scala/org/neo4j/cypher/internal/compiler/v2_3/planDescription/PlanDescriptionArgumentSerializer.scala index 4ff480c09f0a9..999ad4fc6a6b3 100644 --- a/community/cypher/cypher-compiler-2.3/src/main/scala/org/neo4j/cypher/internal/compiler/v2_3/planDescription/PlanDescriptionArgumentSerializer.scala +++ b/community/cypher/cypher-compiler-2.3/src/main/scala/org/neo4j/cypher/internal/compiler/v2_3/planDescription/PlanDescriptionArgumentSerializer.scala @@ -42,6 +42,7 @@ object PlanDescriptionArgumentSerializer { case DbHits(value) => Long.box(value) case _: EntityByIdRhs => arg.toString case Rows(value) => Long.box(value) + case Time(value) => Long.box(value) case EstimatedRows(value) => Double.box(value) case Version(version) => version case Planner(planner) => planner diff --git a/community/cypher/cypher-compiler-2.3/src/test/scala/org/neo4j/cypher/internal/compiler/v2_3/birk/CodeGeneratorTest.scala b/community/cypher/cypher-compiler-2.3/src/test/scala/org/neo4j/cypher/internal/compiler/v2_3/birk/CodeGeneratorTest.scala index 5a018e8170f27..44053f8a8e757 100644 --- a/community/cypher/cypher-compiler-2.3/src/test/scala/org/neo4j/cypher/internal/compiler/v2_3/birk/CodeGeneratorTest.scala +++ b/community/cypher/cypher-compiler-2.3/src/test/scala/org/neo4j/cypher/internal/compiler/v2_3/birk/CodeGeneratorTest.scala @@ -26,6 +26,7 @@ import org.mockito.stubbing.Answer import org.neo4j.collection.primitive.PrimitiveLongIterator import org.neo4j.cypher.internal.compiler.v2_3.ast._ import org.neo4j.cypher.internal.compiler.v2_3.executionplan.InternalExecutionResult +import org.neo4j.cypher.internal.compiler.v2_3.executionplan.ExecutionPlanBuilder.tracer import org.neo4j.cypher.internal.compiler.v2_3.pipes.LazyLabel import org.neo4j.cypher.internal.compiler.v2_3.planner.logical.plans._ import org.neo4j.cypher.internal.compiler.v2_3.planner.{LogicalPlanningTestSupport, SemanticTable} @@ -488,7 +489,7 @@ class CodeGeneratorTest extends CypherFunSuite with LogicalPlanningTestSupport { private def compile(plan: LogicalPlan, params: Map[String, AnyRef] = Map.empty, taskCloser: TaskCloser = new TaskCloser) = { val compiled = generator.generate(plan, newMockedPlanContext, Clock.SYSTEM_CLOCK, mock[SemanticTable]) - compiled.executionResultBuilder(statement, graphDatabaseService, NormalMode, params, taskCloser) + compiled.executionResultBuilder(statement, graphDatabaseService, NormalMode, tracer(NormalMode), params, taskCloser) } /* diff --git a/community/cypher/cypher-compiler-2.3/src/test/scala/org/neo4j/cypher/internal/compiler/v2_3/birk/il/CodeGenSugar.scala b/community/cypher/cypher-compiler-2.3/src/test/scala/org/neo4j/cypher/internal/compiler/v2_3/birk/il/CodeGenSugar.scala index 3d8e4093923a1..5ff109d47e41f 100644 --- a/community/cypher/cypher-compiler-2.3/src/test/scala/org/neo4j/cypher/internal/compiler/v2_3/birk/il/CodeGenSugar.scala +++ b/community/cypher/cypher-compiler-2.3/src/test/scala/org/neo4j/cypher/internal/compiler/v2_3/birk/il/CodeGenSugar.scala @@ -20,9 +20,10 @@ package org.neo4j.cypher.internal.compiler.v2_3.birk.il import org.neo4j.cypher.internal.compiler.v2_3.{TaskCloser, ExecutionMode} -import org.neo4j.cypher.internal.compiler.v2_3.birk.CodeGenerator +import org.neo4j.cypher.internal.compiler.v2_3.birk.{QueryExecutionTracer, CodeGenerator} import org.neo4j.cypher.internal.compiler.v2_3.executionplan.InternalExecutionResult import org.neo4j.cypher.internal.compiler.v2_3.planDescription.InternalPlanDescription +import org.neo4j.function.Supplier import org.neo4j.graphdb.GraphDatabaseService import org.neo4j.graphdb.Result.{ResultRow, ResultVisitor} import org.neo4j.kernel.api.Statement @@ -55,14 +56,16 @@ trait CodeGenSugar extends MockitoSugar { statement: Statement = mock[Statement], graphdb: GraphDatabaseService = null, executionMode: ExecutionMode = null, - description: InternalPlanDescription = null, + description: Supplier[InternalPlanDescription] = null, + queryExecutionTracer: QueryExecutionTracer = null, params: Map[String, Any] = Map.empty): InternalExecutionResult = clazz.getConstructor( classOf[TaskCloser], classOf[Statement], classOf[GraphDatabaseService], classOf[ExecutionMode], - classOf[InternalPlanDescription], + classOf[Supplier[InternalPlanDescription]], + classOf[QueryExecutionTracer], classOf[java.util.Map[String, Object]] - ).newInstance(taskCloser, statement, graphdb, executionMode, description, JavaConversions.mapAsJavaMap(params)) + ).newInstance(taskCloser, statement, graphdb, executionMode, description, queryExecutionTracer, JavaConversions.mapAsJavaMap(params)) } diff --git a/community/cypher/cypher-compiler-2.3/src/test/scala/org/neo4j/cypher/internal/compiler/v2_3/birk/profiling/ProfilingTracerTest.scala b/community/cypher/cypher-compiler-2.3/src/test/scala/org/neo4j/cypher/internal/compiler/v2_3/birk/profiling/ProfilingTracerTest.scala new file mode 100644 index 0000000000000..e16beb663b6a6 --- /dev/null +++ b/community/cypher/cypher-compiler-2.3/src/test/scala/org/neo4j/cypher/internal/compiler/v2_3/birk/profiling/ProfilingTracerTest.scala @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.cypher.internal.compiler.v2_3.birk.profiling + +import org.neo4j.cypher.internal.compiler.v2_3.planDescription.Id +import org.neo4j.cypher.internal.compiler.v2_3.test_helpers.CypherFunSuite + +class ProfilingTracerTest extends CypherFunSuite { + + class Clock extends ProfilingTracer.Clock { + var nanoTime: Long = 0L + + def progress(nanos: Long) { + assert(nanos > 0, "time must move forwards") + nanoTime += nanos + } + } + + test("shouldReportExecutionTimeOfQueryExecution") { + // given + val clock = new Clock + val operatorId = new Id + val tracer = new ProfilingTracer(clock) + val event = tracer.executeQuery(operatorId) + + // when + clock.progress(516) + event.close() + + // then + tracer.timeOf(operatorId) should equal(516) + } + + test("shouldReportDbHitsOfQueryExecution") { + // given + val operatorId = new Id + val tracer = new ProfilingTracer + val event = tracer.executeQuery(operatorId) + + // when + (0 until 516).foreach { _ => + event.dbHit() + } + + event.close() + + // then + tracer.dbHitsOf(operatorId) should equal(516) + } + + test("shouldReportRowsOfQueryExecution") { + // given + val operatorId = new Id + val tracer = new ProfilingTracer + val event = tracer.executeQuery(operatorId) + + // when + (0 until 516).foreach { _ => + event.row() + } + + event.close() + + // then + tracer.rowsOf(operatorId) should equal(516) + + } +} \ No newline at end of file diff --git a/community/cypher/cypher/src/test/scala/org/neo4j/cypher/ProfilerAcceptanceTest.scala b/community/cypher/cypher/src/test/scala/org/neo4j/cypher/ProfilerAcceptanceTest.scala index 03fb327a60fc5..9b0fade8ebbd4 100644 --- a/community/cypher/cypher/src/test/scala/org/neo4j/cypher/ProfilerAcceptanceTest.scala +++ b/community/cypher/cypher/src/test/scala/org/neo4j/cypher/ProfilerAcceptanceTest.scala @@ -25,11 +25,17 @@ import org.neo4j.cypher.internal.compiler.v2_3.commands.expressions.StringHelper import org.neo4j.cypher.internal.compiler.v2_3.executionplan.InternalExecutionResult import org.neo4j.cypher.internal.compiler.v2_3.planDescription.Argument import org.neo4j.cypher.internal.compiler.v2_3.planDescription.InternalPlanDescription.Arguments.{DbHits, Rows} -import org.neo4j.cypher.internal.helpers.TxCounts import org.neo4j.cypher.internal.compiler.v2_3.test_helpers.CreateTempFileTestSupport +import org.neo4j.cypher.internal.helpers.TxCounts class ProfilerAcceptanceTest extends ExecutionEngineFunSuite with CreateTempFileTestSupport with NewPlannerTestSupport { + test("profile with all runtimes") { + val result = profileWithAllPlannersAndRuntimes("MATCH (n) RETURN n") + val executionPlanDescription = result.executionPlanDescription() + println(executionPlanDescription) + } + test("match n where n-[:FOO]->() return *") { //GIVEN relate( createNode(), createNode(), "FOO") @@ -125,7 +131,7 @@ class ProfilerAcceptanceTest extends ExecutionEngineFunSuite with CreateTempFile createNode() //GIVEN - val result = profileWithAllPlanners("MATCH n RETURN n.foo") + val result = profileWithAllPlannersAndRuntimes("MATCH n RETURN n.foo") //WHEN THEN assertRows(1)(result)("ColumnFilter") @@ -275,7 +281,7 @@ class ProfilerAcceptanceTest extends ExecutionEngineFunSuite with CreateTempFile } test("should show expand without types in a simple form") { - val a = profileWithAllPlanners("match n-->() return *") + val a = profileWithAllPlannersAndRuntimes("match n-->() return *") a.executionPlanDescription().toString should include("()<--(n)") } @@ -314,9 +320,9 @@ class ProfilerAcceptanceTest extends ExecutionEngineFunSuite with CreateTempFile result } - //TODO change to executeWithAllPlanners when PROFILING is supported in compiled plans - def profileWithAllPlanners(q: String, params: (String, Any)*): InternalExecutionResult = - profileWithPlanner(executeWithAllPlannersOnInterpretedRuntime(_,_:_*), q, params:_*) + def profileWithAllPlanners(q: String, params: (String, Any)*): InternalExecutionResult = profileWithPlanner(executeWithAllPlanners(_,_:_*), q, params:_*) + + def profileWithAllPlannersAndRuntimes(q: String, params: (String, Any)*): InternalExecutionResult = profileWithPlanner(executeWithAllPlannersAndRuntimes(_,_:_*), q, params:_*) override def profile(q: String, params: (String, Any)*): InternalExecutionResult = fail("Don't use profile together in ProfilerAcceptanceTest") diff --git a/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/RewindableExecutionResult.scala b/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/RewindableExecutionResult.scala index 6e97d6212872f..d0bf343f137c9 100644 --- a/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/RewindableExecutionResult.scala +++ b/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/RewindableExecutionResult.scala @@ -29,6 +29,7 @@ import org.neo4j.cypher.internal.compiler.v2_3.planDescription.InternalPlanDescr import org.neo4j.cypher.internal.compiler.v2_3.planDescription.InternalPlanDescription.Arguments.{Planner, Runtime} import org.neo4j.cypher.internal.compiler.v2_3.{ExecutionMode => ExecutionModev2_3, PipeExecutionResult, PlannerName, RuntimeName, TaskCloser} import org.neo4j.cypher.{ExecutionResult, InternalException} +import org.neo4j.function.Suppliers.singleton import org.neo4j.graphdb.QueryExecutionType.QueryType import org.neo4j.graphdb.Result.ResultVisitor import org.neo4j.kernel.api.Statement diff --git a/community/cypher/cypher/src/test/scala/org/neo4j/cypher/tempgen b/community/cypher/cypher/src/test/scala/org/neo4j/cypher/tempgen new file mode 100644 index 0000000000000..3ba67506242d4 --- /dev/null +++ b/community/cypher/cypher/src/test/scala/org/neo4j/cypher/tempgen @@ -0,0 +1,148 @@ +org.neo4j.cypher.internal.compiler.v2_3.birk.Javac$CompilationError: package org.neo4j.cypher.internal.compiler.v2_3.birk.generated; + +import org.neo4j.helpers.collection.Visitor; +import org.neo4j.function.Supplier; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.kernel.api.Statement; +import org.neo4j.kernel.api.exceptions.KernelException; +import org.neo4j.kernel.api.ReadOperations; +import org.neo4j.cypher.internal.compiler.v2_3.birk.ResultRowImpl; +import org.neo4j.cypher.internal.compiler.v2_3.executionplan.CompiledExecutionResult; +import org.neo4j.graphdb.Result.ResultRow; +import org.neo4j.graphdb.Result.ResultVisitor; +import org.neo4j.graphdb.Result; +import org.neo4j.graphdb.Transaction; +import org.neo4j.cypher.internal.compiler.v2_3.planDescription.Id; +import org.neo4j.cypher.internal.compiler.v2_3.planDescription.InternalPlanDescription; +import org.neo4j.cypher.internal.compiler.v2_3.ExecutionMode; +import org.neo4j.cypher.internal.compiler.v2_3.executionplan.CompletionListener; +import org.neo4j.cypher.internal.compiler.v2_3.birk.QueryExecutionTracer; +import org.neo4j.cypher.internal.compiler.v2_3.birk.QueryExecutionEvent; +import java.util.Map; +import java.util.List; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.neo4j.collection.primitive.Primitive; +import org.neo4j.collection.primitive.PrimitiveLongIntMap; +import org.neo4j.collection.primitive.PrimitiveLongIterator; +import org.neo4j.collection.primitive.hopscotch.LongKeyIntValueTable; + +public class GeneratedExecutionPlan1 extends CompiledExecutionResult +{ +private final ReadOperations ro; +private final GraphDatabaseService db; +private final Map params; +private final QueryExecutionTracer tracer; + +public static final Id OP1_NodeHashJoin; +public static final Id OP2_AllNodesScan; +public static final Id OP4_AllNodesScan; +public static final Id OP3_ProduceResult; + +public GeneratedExecutionPlan1( CompletionListener completion, Statement statement, GraphDatabaseService db, ExecutionMode executionMode, Supplier description, QueryExecutionTracer tracer, Map params ) +{ + super( completion, statement, executionMode, description ); + this.ro = statement.readOperations(); + this.db = db; + this.tracer = tracer; + this.params = params; +} + + +@Override +public List javaColumns( ) +{ +return Arrays.asList( "a" ); +} + +@Override +public void accept(final ResultVisitor visitor) +{ +final ResultRowImpl row = new ResultRowImpl(db); + + +try +{ +PrimitiveLongIntMap v2 = null; +try ( QueryExecutionEvent event_OP1_NodeHashJoin = tracer.executeOperator( OP1_NodeHashJoin ) ) +{ +v2 = m1(); +} +try ( QueryExecutionEvent event_OP4_AllNodesScan = tracer.executeOperator( OP4_AllNodesScan ) ) +{ +PrimitiveLongIterator v4Iter = ro.nodesGetAll(); +event_OP4_AllNodesScan.dbHit(); +while ( v4Iter.hasNext() ) +{ +event_OP4_AllNodesScan.dbHit(); +event_OP4_AllNodesScan.row(); +final long v4 = v4Iter.next(); + + +try ( QueryExecutionEvent event_OP1_NodeHashJoin = tracer.executeOperator( OP1_NodeHashJoin ) ) +{ +int v3 = v2.get( v4 ); +if ( v3 != LongKeyIntValueTable.NULL ) +{ +for ( int i = 0; i < v3; i++ ) +{ +event_OP1_NodeHashJoin.row(); +row.setNode("a", v4); + + +try ( QueryExecutionEvent event_OP3_ProduceResult = tracer.executeOperator( OP3_ProduceResult ) ) +{ +if ( !visitor.visit(row) ) +{ +return; +} +event_OP3_ProduceResult.row(); +} +} +} +} +} +} +success(); +} +catch (Exception e) +{ +//TODO proper error handling +e.printStackTrace(); +throw new RuntimeException( e ); +} +finally +{ +close(); +} +} +private PrimitiveLongIntMap m1() throws KernelException +{ + +final PrimitiveLongIntMap v2 = Primitive.longIntMap(); +try ( QueryExecutionEvent event_OP2_AllNodesScan = tracer.executeOperator( OP2_AllNodesScan ) ) +{ +PrimitiveLongIterator v1Iter = ro.nodesGetAll(); +event_OP2_AllNodesScan.dbHit(); +while ( v1Iter.hasNext() ) +{ +event_OP2_AllNodesScan.dbHit(); +event_OP2_AllNodesScan.row(); +final long v1 = v1Iter.next(); + +int count = v2.get( v1 ); +if ( count == LongKeyIntValueTable.NULL ) +{ +v2.put( v1, 1 ); +} +else +{ +v2.put( v1, count + 1 ); +} +} +} + +return v2; +}