Skip to content

Commit

Permalink
Make sure all tests pass
Browse files Browse the repository at this point in the history
  • Loading branch information
systay committed Feb 7, 2017
1 parent 9461d18 commit 573180e
Show file tree
Hide file tree
Showing 40 changed files with 429 additions and 424 deletions.
@@ -1,3 +1,22 @@
/*
* Copyright (c) 2002-2017 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.cypher.internal.compiler.v3_2

import java.time.Clock
Expand All @@ -8,29 +27,38 @@ import org.neo4j.cypher.internal.compiler.v3_2.phases.CompilerContext
import org.neo4j.cypher.internal.compiler.v3_2.planner.logical.{Metrics, MetricsFactory, QueryGraphSolver}
import org.neo4j.cypher.internal.compiler.v3_2.spi.PlanContext
import org.neo4j.cypher.internal.frontend.v3_2.InputPosition
import org.neo4j.cypher.internal.frontend.v3_2.phases.{CompilationPhaseTracer, InternalNotificationLogger, Monitors}
import org.neo4j.cypher.internal.frontend.v3_2.phases.{BaseContext, CompilationPhaseTracer, InternalNotificationLogger, Monitors}

trait ContextCreator {
trait ContextCreator[Context <: BaseContext] {
def create(tracer: CompilationPhaseTracer,
notificationLogger: InternalNotificationLogger,
planContext: PlanContext,
queryText: String,
offset: Option[InputPosition]): CompilerContext
offset: Option[InputPosition],
monitors: Monitors,
createFingerprintReference: Option[PlanFingerprint] => PlanFingerprintReference,
typeConverter: RuntimeTypeConverter,
metricsFactory: MetricsFactory,
queryGraphSolver: QueryGraphSolver,
config: CypherCompilerConfiguration,
updateStrategy: UpdateStrategy,
clock: Clock): Context
}

class CommunityContextCreator(monitors: Monitors,
createFingerprintReference: Option[PlanFingerprint] => PlanFingerprintReference,
typeConverter: RuntimeTypeConverter,
metricsFactory: MetricsFactory,
queryGraphSolver: QueryGraphSolver,
config: CypherCompilerConfiguration,
updateStrategy: UpdateStrategy,
clock: Clock) extends ContextCreator {
object CommunityContextCreator extends ContextCreator[CompilerContext] {
override def create(tracer: CompilationPhaseTracer,
notificationLogger: InternalNotificationLogger,
planContext: PlanContext,
queryText: String,
offset: Option[InputPosition]): CompilerContext = {
offset: Option[InputPosition],
monitors: Monitors,
createFingerprintReference: Option[PlanFingerprint] => PlanFingerprintReference,
typeConverter: RuntimeTypeConverter,
metricsFactory: MetricsFactory,
queryGraphSolver: QueryGraphSolver,
config: CypherCompilerConfiguration,
updateStrategy: UpdateStrategy,
clock: Clock): CompilerContext = {
val exceptionCreator = new SyntaxExceptionCreator(queryText, offset)

val metrics: Metrics = if (planContext == null)
Expand All @@ -41,5 +69,4 @@ class CommunityContextCreator(monitors: Monitors,
new CompilerContext(exceptionCreator, tracer, notificationLogger, planContext, typeConverter, createFingerprintReference,
monitors, metrics, queryGraphSolver, config, updateStrategy, clock)
}

}
Expand Up @@ -36,7 +36,7 @@ import org.neo4j.cypher.internal.frontend.v3_2.helpers.rewriting.RewriterStepSeq
import org.neo4j.cypher.internal.frontend.v3_2.phases.{CompilationPhaseTracer, InternalNotificationLogger, Monitors}
import org.neo4j.cypher.internal.compiler.v3_2.phases.CompilerContext

case class CypherCompiler(createExecutionPlan: Transformer[CompilerContext],
case class CypherCompiler[Context <: CompilerContext](createExecutionPlan: Transformer[Context],
astRewriter: ASTRewriter,
cacheAccessor: CacheAccessor[Statement, ExecutionPlan],
planCacheFactory: () => LFUCache[Statement, ExecutionPlan],
Expand All @@ -50,7 +50,7 @@ case class CypherCompiler(createExecutionPlan: Transformer[CompilerContext],
config: CypherCompilerConfiguration,
updateStrategy: UpdateStrategy,
clock: Clock,
contextCreation: ContextCreator) {
contextCreation: ContextCreator[Context]) {

def planQuery(queryText: String,
context: PlanContext,
Expand All @@ -66,7 +66,9 @@ case class CypherCompiler(createExecutionPlan: Transformer[CompilerContext],
planContext: PlanContext,
offset: Option[InputPosition] = None,
tracer: CompilationPhaseTracer): (ExecutionPlan, Map[String, Any]) = {
val context = contextCreation.create(tracer, notificationLogger, planContext, input.queryText, input.startPosition)
val context: Context = contextCreation.create(tracer, notificationLogger, planContext, input.queryText,
input.startPosition, monitors, createFingerprintReference, typeConverter, metricsFactory,
queryGraphSolver, config, updateStrategy, clock)
val preparedCompilationState = prepareForCaching.transform(input, context)
val cache = provideCache(cacheAccessor, cacheMonitor, planContext)
val isStale = (plan: ExecutionPlan) => plan.isStale(planContext.txIdProvider, planContext.statistics)
Expand All @@ -86,28 +88,29 @@ case class CypherCompiler(createExecutionPlan: Transformer[CompilerContext],
val plannerName = PlannerName(plannerNameText)
val startState = CompilationState(queryText, offset, plannerName)
//TODO: these nulls are a short cut
val context = contextCreation.create(tracer, notificationLogger, planContext = null, rawQueryText, offset)
val context = contextCreation.create(tracer, notificationLogger, planContext = null, rawQueryText, offset, monitors,
createFingerprintReference, typeConverter, metricsFactory, queryGraphSolver, config, updateStrategy, clock)
CompilationPhases.parsing(sequencer).transform(startState, context)
}

val irConstruction: Transformer[CompilerContext] =
val irConstruction: Transformer[Context] =
ResolveTokens andThen
CreatePlannerQuery.adds[UnionQuery] andThen
OptionalMatchRemover

val prepareForCaching: Transformer[CompilerContext] =
val prepareForCaching: Transformer[Context] =
RewriteProcedureCalls andThen
ProcedureDeprecationWarnings andThen
ProcedureWarnings

val costBasedPlanning: Transformer[CompilerContext] =
val costBasedPlanning: Transformer[Context] =
QueryPlanner().adds[LogicalPlan] andThen
PlanRewriter(sequencer) andThen
If(_.unionQuery.readOnly) (
CheckForUnresolvedTokens
)

val planAndCreateExecPlan: Transformer[CompilerContext] =
val planAndCreateExecPlan: Transformer[Context] =
ProcedureCallOrSchemaCommandPlanBuilder andThen
If(_.maybeExecutionPlan.isEmpty)(
CompilationPhases.lateAstRewriting andThen
Expand Down
Expand Up @@ -23,14 +23,14 @@ import java.time.Clock

import org.neo4j.cypher.internal.compiler.v3_2.executionplan._
import org.neo4j.cypher.internal.compiler.v3_2.helpers.RuntimeTypeConverter
import org.neo4j.cypher.internal.compiler.v3_2.phases.CompilerContext
import org.neo4j.cypher.internal.compiler.v3_2.planner.logical._
import org.neo4j.cypher.internal.compiler.v3_2.phases.{CompilerContext, Transformer}
import org.neo4j.cypher.internal.compiler.v3_2.planner.logical.idp._
import org.neo4j.cypher.internal.frontend.v3_2.ast.Statement
import org.neo4j.cypher.internal.frontend.v3_2.helpers.rewriting.RewriterStepSequencer
import org.neo4j.cypher.internal.frontend.v3_2.phases.Monitors

object CypherCompilerFactory {
class CypherCompilerFactory[C <: CompilerContext, T <: Transformer[C]] {
val monitorTag = "cypher3.2"

def costBasedCompiler(config: CypherCompilerConfiguration,
Expand All @@ -41,7 +41,8 @@ object CypherCompilerFactory {
runtimeName: Option[RuntimeName],
updateStrategy: Option[UpdateStrategy],
typeConverter: RuntimeTypeConverter,
runtimeBuilder: RuntimeBuilder[CompilerContext]): CypherCompiler = {
runtimeBuilder: RuntimeBuilder[T],
contextCreator: ContextCreator[C]): CypherCompiler[C] = {
val rewriter = new ASTRewriter(rewriterSequencer)
val metricsFactory = CachedMetricsFactory(SimpleMetricsFactory)

Expand All @@ -60,9 +61,6 @@ object CypherCompilerFactory {
val cacheMonitor = monitors.newMonitor[AstCacheMonitor](monitorTag)
val cache = new MonitoringCacheAccessor[Statement, ExecutionPlan](cacheMonitor)

val contextCreator = new CommunityContextCreator(monitors, createFingerprintReference, typeConverter,
metricsFactory, queryGraphSolver, config, actualUpdateStrategy, clock)

CypherCompiler(runtimePipeline, rewriter, cache, planCacheFactory, cacheMonitor, monitors, rewriterSequencer,
createFingerprintReference, typeConverter, metricsFactory, queryGraphSolver, config, actualUpdateStrategy, clock,
contextCreator)
Expand Down
Expand Up @@ -22,12 +22,13 @@ package org.neo4j.cypher.internal.compiler.v3_2
import org.neo4j.cypher.internal.compiler.v3_2.phases._
import org.neo4j.cypher.internal.frontend.v3_2.InvalidArgumentException

trait RuntimeBuilder[C <: CompilerContext] {
def create(runtimeName: Option[RuntimeName], useErrorsOverWarnings: Boolean): Transformer[C]
trait RuntimeBuilder[T <: Transformer[_]] {
def create(runtimeName: Option[RuntimeName], useErrorsOverWarnings: Boolean): T
}

object CommunityRuntimeBuilder extends RuntimeBuilder[CompilerContext] {
def create(runtimeName: Option[RuntimeName], useErrorsOverWarnings: Boolean): Transformer[CompilerContext] = runtimeName match {
object CommunityRuntimeBuilder extends RuntimeBuilder[Transformer[CompilerContext]] {
override def create(runtimeName: Option[RuntimeName], useErrorsOverWarnings: Boolean): Transformer[CompilerContext] =
runtimeName match {
case None | Some(InterpretedRuntimeName) =>
BuildInterpretedExecutionPlan

Expand Down
Expand Up @@ -21,7 +21,6 @@ package org.neo4j.cypher.internal.compiler.v3_2.planDescription

import org.neo4j.cypher.internal.compiler.v3_2.planDescription.InternalPlanDescription.Arguments._

import scala.Predef
import scala.collection.{Map, mutable}

object renderAsTreeTable extends (InternalPlanDescription => String) {
Expand All @@ -35,16 +34,16 @@ object renderAsTreeTable extends (InternalPlanDescription => String) {
val MAX_VARIABLE_COLUMN_WIDTH = 100
private val OTHER = "Other"
private val HEADERS = Seq(OPERATOR, ESTIMATED_ROWS, ROWS, HITS, TIME, VARIABLES, OTHER)
val newLine = System.lineSeparator()
private val newLine = System.lineSeparator()

def apply(plan: InternalPlanDescription): String = {

implicit val columns = new mutable.HashMap[String, Int]()
val lines = accumulate(plan)

def compactLine(line: Line, previous: Seq[(Line, CompactedLine)]) = {
val repeatedVariables = if (previous.size > 0) previous.head._1.variables.intersect(line.variables) else Set.empty[String]
new CompactedLine(line, repeatedVariables)
val repeatedVariables = if (previous.nonEmpty) previous.head._1.variables.intersect(line.variables) else Set.empty[String]
CompactedLine(line, repeatedVariables)
}

val compactedLines = lines.reverse.foldLeft(Seq.empty[(Line, CompactedLine)]) { (acc, line) =>
Expand All @@ -63,7 +62,7 @@ object renderAsTreeTable extends (InternalPlanDescription => String) {
val result = new StringBuilder((2 + newLine.length + headers.map(width).sum) * (lines.size * 2 + 3))

def pad(width:Int, char:String=" ") =
for (i <- 1 to width) result.append(char)
for (_ <- 1 to width) result.append(char)
def divider(line:LineDetails = null) = {
for (header <- headers) {
if (line != null && header == OPERATOR && line.connection.isDefined) {
Expand Down Expand Up @@ -188,25 +187,25 @@ case class CompactedLine(line: Line, repeated: Set[String]) extends LineDetails
val varSep = ", "
val typeSep = " -- "
val suffix = ", ..."
val formattedVariables = formatVariables(renderAsTreeTable.MAX_VARIABLE_COLUMN_WIDTH)
val formattedVariables: String = formatVariables(renderAsTreeTable.MAX_VARIABLE_COLUMN_WIDTH)

def apply(key: String): Justified = if (key == renderAsTreeTable.VARIABLES)
Left(formattedVariables)
else
line(key)

def connection = line.connection
def connection: Option[String] = line.connection

private def formattedVars(vars: List[String], prefix: String = "") = vars match {
case v :: Nil => List(prefix + v)
case v :: tail => List(prefix + v) ++ tail.map(v => varSep + v)
case _ => vars
}

def formatVariables(length: Int) = {
def formatVariables(length: Int): String = {
val newVars = (line.variables -- repeated).toList.sorted.map(PlanDescriptionArgumentSerializer.removeGeneratedNames)
val oldVars = repeated.toList.sorted.map(PlanDescriptionArgumentSerializer.removeGeneratedNames)
val all = if(newVars.length > 0)
val all = if(newVars.nonEmpty)
formattedVars(newVars) ++ formattedVars(oldVars, typeSep)
else
formattedVars(oldVars)
Expand Down Expand Up @@ -239,7 +238,7 @@ case class CompactedLine(line: Line, repeated: Set[String]) extends LineDetails
}

sealed abstract class Justified(text:String) {
def length = text.length
def length: Int = text.length
}
case class Left(text:String) extends Justified(text)
case class Right(text:String) extends Justified(text)
Expand Down
Expand Up @@ -59,7 +59,7 @@ protected QueryExecutionEngine createEngine( Dependencies deps, GraphDatabaseAPI
LogProvider logProvider = logService.getInternalLogProvider();
CommunityCompatibilityFactory compatibilityFactory =
new CommunityCompatibilityFactory( queryService, kernelAPI, monitors, logProvider );

deps.satisfyDependencies( compatibilityFactory );
return new ExecutionEngine( queryService, logProvider, compatibilityFactory);
}
}
Expand Up @@ -22,7 +22,7 @@ package org.neo4j.cypher.internal
import org.neo4j.cypher.internal.compatibility.v2_3.helpers._
import org.neo4j.cypher.internal.compatibility.v3_1.helpers._
import org.neo4j.cypher.internal.compatibility.{v2_3, v3_1, _}
import org.neo4j.cypher.internal.compiler.v3_2.CypherCompilerConfiguration
import org.neo4j.cypher.internal.compiler.v3_2.{CommunityContextCreator, CommunityRuntimeBuilder, CypherCompilerConfiguration}
import org.neo4j.cypher.internal.frontend.v3_2.InvalidArgumentException
import org.neo4j.cypher.{CypherCodeGenMode, CypherPlanner, CypherRuntime, CypherUpdateStrategy}
import org.neo4j.helpers.Clock
Expand All @@ -43,7 +43,7 @@ trait CompatibilityFactory {

def create(spec: PlannerSpec_v3_1, config: CypherCompilerConfiguration): v3_1.Compatibility

def create(spec: PlannerSpec_v3_2, config: CypherCompilerConfiguration): v3_2.Compatibility
def create(spec: PlannerSpec_v3_2, config: CypherCompilerConfiguration): v3_2.Compatibility[_]
}

class CommunityCompatibilityFactory(graph: GraphDatabaseQueryService, kernelAPI: KernelAPI, kernelMonitors: KernelMonitors,
Expand All @@ -65,28 +65,29 @@ class CommunityCompatibilityFactory(graph: GraphDatabaseQueryService, kernelAPI:
v3_1.CostCompatibility(graph, as3_1(config), CompilerEngineDelegator.CLOCK, kernelMonitors, kernelAPI, log, spec.planner, spec.runtime, spec.updateStrategy)
}

override def create(spec: PlannerSpec_v3_2, config: CypherCompilerConfiguration): v3_2.Compatibility =
override def create(spec: PlannerSpec_v3_2, config: CypherCompilerConfiguration): v3_2.Compatibility[_] =
(spec.planner, spec.runtime) match {
case (CypherPlanner.rule, _) =>
throw new InvalidArgumentException("The rule planner is no longer a valid planner option in Neo4j 3.2. If you need to use it, please compatibility mode Cypher 3.1")
case (_, CypherRuntime.compiled) =>
throw new InvalidArgumentException("The compiled runtime is only available in the Enterprise version of Neo4j")
case _ =>
v3_2.CostCompatibility(config, CompilerEngineDelegator.CLOCK, kernelMonitors, kernelAPI, log, spec.planner, spec.runtime, spec.codeGenMode, spec.updateStrategy)
v3_2.CostCompatibility(config, CompilerEngineDelegator.CLOCK, kernelMonitors, kernelAPI, log,
spec.planner, spec.runtime, spec.updateStrategy, CommunityRuntimeBuilder, CommunityContextCreator)
}
}

class CompatibilityCache(factory: CompatibilityFactory) extends CompatibilityFactory {
private val cache_v2_3 = new mutable.HashMap[PlannerSpec_v2_3, v2_3.Compatibility]
private val cache_v3_1 = new mutable.HashMap[PlannerSpec_v3_1, v3_1.Compatibility]
private val cache_v3_2 = new mutable.HashMap[PlannerSpec_v3_2, v3_2.Compatibility]
private val cache_v3_2 = new mutable.HashMap[PlannerSpec_v3_2, v3_2.Compatibility[_]]

override def create(spec: PlannerSpec_v2_3, config: CypherCompilerConfiguration): v2_3.Compatibility =
cache_v2_3.getOrElseUpdate(spec, factory.create(spec, config))

override def create(spec: PlannerSpec_v3_1, config: CypherCompilerConfiguration): v3_1.Compatibility =
cache_v3_1.getOrElseUpdate(spec, factory.create(spec, config))

override def create(spec: PlannerSpec_v3_2, config: CypherCompilerConfiguration): v3_2.Compatibility =
override def create(spec: PlannerSpec_v3_2, config: CypherCompilerConfiguration): v3_2.Compatibility[_] =
cache_v3_2.getOrElseUpdate(spec, factory.create(spec, config))
}
Expand Up @@ -23,7 +23,7 @@ import org.neo4j.cypher.internal._
import org.neo4j.cypher.internal.compatibility._
import org.neo4j.cypher.internal.compiler.v3_2
import org.neo4j.cypher.internal.compiler.v3_2.executionplan.{LegacyNodeIndexUsage, LegacyRelationshipIndexUsage, SchemaIndexScanUsage, SchemaIndexSeekUsage, ExecutionPlan => ExecutionPlan_v3_2}
import org.neo4j.cypher.internal.compiler.v3_2.phases.CompilationState
import org.neo4j.cypher.internal.compiler.v3_2.phases.{CompilationState, CompilerContext}
import org.neo4j.cypher.internal.compiler.v3_2.{InfoLogger, ExplainMode => ExplainModev3_2, NormalMode => NormalModev3_2, ProfileMode => ProfileModev3_2}
import org.neo4j.cypher.internal.frontend.v3_2.helpers.rewriting.RewriterStepSequencer
import org.neo4j.cypher.internal.frontend.v3_2.phases.{CompilationPhaseTracer, RecordingNotificationLogger}
Expand All @@ -38,7 +38,7 @@ import org.neo4j.logging.Log

import scala.util.Try

trait Compatibility {
trait Compatibility[C <: CompilerContext] {
val queryCacheSize: Int
val kernelMonitors: KernelMonitors
val kernelAPI: KernelAPI
Expand All @@ -50,7 +50,7 @@ trait Compatibility {
if (assertionsEnabled()) newValidating else newPlain
}

protected val compiler: v3_2.CypherCompiler
protected val compiler: v3_2.CypherCompiler[C]

implicit val executionMonitor: QueryExecutionMonitor = kernelMonitors.newMonitor(classOf[QueryExecutionMonitor])

Expand Down

0 comments on commit 573180e

Please sign in to comment.