Skip to content

Commit

Permalink
Merge pull request #4327 from boggle/scala_2.11
Browse files Browse the repository at this point in the history
Build scala 2.11 module of cypher compiler 2.2
  • Loading branch information
Stefan Plantikow committed Apr 9, 2015
2 parents 6dbee96 + 7f5d616 commit 1f64943
Show file tree
Hide file tree
Showing 49 changed files with 743 additions and 315 deletions.
54 changes: 54 additions & 0 deletions community/cypher/cypher-compiler-2.2/pom.xml
Expand Up @@ -22,6 +22,8 @@

<properties>
<version-package>cypher.internal.compiler.v2_2</version-package>
<scala.version>2.10.5</scala.version>
<scala.binary.version>2.10</scala.binary.version>
</properties>

<licenses>
Expand All @@ -46,6 +48,21 @@

<build>
<plugins>

<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<configuration>
<scalaVersion>${scala.version}</scalaVersion>
<scalaCompatVersion>${scala.binary.version}</scalaCompatVersion>
<args>
<arg>-Xmax-classfile-name</arg>
<arg>100</arg>
<arg>-Xlint</arg>
</args>
</configuration>
</plugin>

<plugin>
<groupId>org.scalastyle</groupId>
<artifactId>scalastyle-maven-plugin</artifactId>
Expand All @@ -59,9 +76,46 @@

<!-- scala -->

<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
</dependency>

<dependency>
<artifactId>scala-reflect</artifactId>
<groupId>org.scala-lang</groupId>
<version>${scala.version}</version>
</dependency>

<!-- scala test dependencies -->

<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_2.10</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<artifactId>scala-library</artifactId>
<groupId>org.scala-lang</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.scalautils</groupId>
<artifactId>scalautils_2.10</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<artifactId>scala-library</artifactId>
<groupId>org.scala-lang</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.scalacheck</groupId>
<artifactId>scalacheck_2.10</artifactId>
<scope>test</scope>
</dependency>

<!-- neo4j -->
Expand Down
Expand Up @@ -24,7 +24,7 @@ import org.neo4j.cypher.internal.compiler.v2_2.ast.rewriters._
import org.neo4j.cypher.internal.compiler.v2_2.ast._
import org.neo4j.cypher.internal.compiler.v2_2.tracing.rewriters.{ApplyRewriter, RewriterCondition, RewriterStepSequencer}

class ASTRewriter(rewritingMonitor: AstRewritingMonitor, shouldExtractParameters: Boolean = true) {
class ASTRewriter(rewriterSequencer: (String) => RewriterStepSequencer, rewritingMonitor: AstRewritingMonitor, shouldExtractParameters: Boolean = true) {

import org.neo4j.cypher.internal.compiler.v2_2.tracing.rewriters.RewriterStep._

Expand All @@ -36,7 +36,7 @@ class ASTRewriter(rewritingMonitor: AstRewritingMonitor, shouldExtractParameters
else
(Rewriter.lift(PartialFunction.empty), Map.empty[String, Any])

val contract = RewriterStepSequencer.newDefault("ASTRewriter")(
val contract = rewriterSequencer("ASTRewriter")(
enableCondition(noReferenceEqualityAmongIdentifiers),
enableCondition(containsNoNodesOfType[UnaliasedReturnItem]),
enableCondition(orderByOnlyOnIdentifiers),
Expand Down
Expand Up @@ -25,8 +25,10 @@ import org.neo4j.cypher.internal.compiler.v2_2.ast.rewriters.{normalizeReturnCla
import org.neo4j.cypher.internal.compiler.v2_2.executionplan._
import org.neo4j.cypher.internal.compiler.v2_2.parser.{CypherParser, ParserMonitor}
import org.neo4j.cypher.internal.compiler.v2_2.planner._
import org.neo4j.cypher.internal.compiler.v2_2.planner.logical.{CachedMetricsFactory, SimpleMetricsFactory}
import org.neo4j.cypher.internal.compiler.v2_2.planner.logical.plans.rewriter.LogicalPlanRewriter
import org.neo4j.cypher.internal.compiler.v2_2.planner.logical.{DefaultQueryPlanner, CachedMetricsFactory, SimpleMetricsFactory}
import org.neo4j.cypher.internal.compiler.v2_2.spi.PlanContext
import org.neo4j.cypher.internal.compiler.v2_2.tracing.rewriters.{RewriterStepSequencer, PlainRewriterStepSequencer}
import org.neo4j.graphdb.GraphDatabaseService
import org.neo4j.helpers.Clock

Expand Down Expand Up @@ -65,15 +67,17 @@ object CypherCompilerFactory {

def costBasedCompiler(graph: GraphDatabaseService, queryCacheSize: Int, statsDivergenceThreshold: Double,
queryPlanTTL: Long, clock: Clock, monitors: Monitors,
logger: InfoLogger, plannerName: CostBasedPlannerName): CypherCompiler = {
logger: InfoLogger, plannerName: CostBasedPlannerName,
rewriterSequencer: (String) => RewriterStepSequencer): CypherCompiler = {
val parser = new CypherParser(monitors.newMonitor[ParserMonitor[Statement]](monitorTag))
val checker = new SemanticChecker(monitors.newMonitor[SemanticCheckMonitor](monitorTag))
val rewriter = new ASTRewriter(monitors.newMonitor[AstRewritingMonitor](monitorTag))
val rewriter = new ASTRewriter(rewriterSequencer, monitors.newMonitor[AstRewritingMonitor](monitorTag))
val planBuilderMonitor = monitors.newMonitor[NewLogicalPlanSuccessRateMonitor](monitorTag)
val planningMonitor = monitors.newMonitor[PlanningMonitor](monitorTag)
val metricsFactory = CachedMetricsFactory(SimpleMetricsFactory)
val planner = CostBasedPipeBuilderFactory(monitors, metricsFactory, planningMonitor, clock, plannerName = plannerName)
val pipeBuilder = new LegacyVsNewPipeBuilder(new LegacyPipeBuilder(monitors), planner, planBuilderMonitor)
val queryPlanner = new DefaultQueryPlanner(LogicalPlanRewriter(rewriterSequencer))
val planner = CostBasedPipeBuilderFactory(monitors, metricsFactory, planningMonitor, clock, queryPlanner = queryPlanner, rewriterSequencer = rewriterSequencer, plannerName = plannerName)
val pipeBuilder = new LegacyVsNewPipeBuilder(new LegacyPipeBuilder(monitors, rewriterSequencer), planner, planBuilderMonitor)
val execPlanBuilder = new ExecutionPlanBuilder(graph, statsDivergenceThreshold, queryPlanTTL, clock, pipeBuilder)
val planCacheFactory = () => new LRUCache[Statement, ExecutionPlan](queryCacheSize)
monitors.addMonitorListener(logStalePlanRemovalMonitor(logger), monitorTag)
Expand All @@ -84,11 +88,12 @@ object CypherCompilerFactory {
}

def ruleBasedCompiler(graph: GraphDatabaseService, queryCacheSize: Int, statsDivergenceThreshold: Double,
queryPlanTTL: Long, clock: Clock, monitors: Monitors): CypherCompiler = {
queryPlanTTL: Long, clock: Clock, monitors: Monitors,
rewriterSequencer: (String) => RewriterStepSequencer): CypherCompiler = {
val parser = new CypherParser(monitors.newMonitor[ParserMonitor[ast.Statement]](monitorTag))
val checker = new SemanticChecker(monitors.newMonitor[SemanticCheckMonitor](monitorTag))
val rewriter = new ASTRewriter(monitors.newMonitor[AstRewritingMonitor](monitorTag))
val pipeBuilder = new LegacyPipeBuilder(monitors)
val rewriter = new ASTRewriter(rewriterSequencer, monitors.newMonitor[AstRewritingMonitor](monitorTag))
val pipeBuilder = new LegacyPipeBuilder(monitors, rewriterSequencer)

val execPlanBuilder = new ExecutionPlanBuilder(graph, statsDivergenceThreshold, queryPlanTTL, clock, pipeBuilder)
val planCacheFactory = () => new LRUCache[Statement, ExecutionPlan](queryCacheSize)
Expand Down
Expand Up @@ -74,13 +74,13 @@ class InvalidInputErrorFormatter extends DefaultInvalidInputErrorFormatter {
}

private def findProperLabelMatcher(path: MatcherPath, errorIndex: Int) : Matcher = {
val elements = unfoldRight(path) { p => if (p == null) None else Some(p.element, p.parent) }.reverse
val elements = unfoldRight(path) { p => if (p == null) None else Some((p.element, p.parent)) }.reverse

val matcher = for (element <- elements.takeWhile(!_.matcher.isInstanceOf[TestNotMatcher]).find(e => {
e.startIndex == errorIndex && e.matcher.hasCustomLabel
})) yield element.matcher

matcher.getOrElse(null)
matcher.orNull
}

private def unfoldRight[A, B](seed: B)(f: B => Option[(A, B)]): List[A] = f(seed) match {
Expand Down
Expand Up @@ -39,6 +39,7 @@ case class Yes[T](values: Seq[T]) extends Maybe[T] {
def map[B](f: T => B): Maybe[B] = Yes(values.map(f))

def seqMap[B](f: (Seq[T]) => Seq[B]): Maybe[B] = Yes(f(values))

def getValuesOr[B >: T](f: => Seq[B]) = values
}

Expand All @@ -55,5 +56,5 @@ case class No(messages: Seq[String]) extends Maybe[Nothing] {

def seqMap[B](f: (Seq[Nothing]) => Seq[B]): Maybe[B] = this

def getValuesOr[Nothing](f: => Seq[Nothing]) = f
def getValuesOr[B >: Nothing](f: => Seq[B]) = f
}
Expand Up @@ -53,7 +53,7 @@ class OptionSemanticChecking[A](val option: Option[A]) extends AnyVal {

class TraversableOnceSemanticChecking[A](val traversable: TraversableOnce[A]) extends AnyVal {
def foldSemanticCheck(check: A => SemanticCheck): SemanticCheck = state => traversable.foldLeft(SemanticCheckResult.success(state)) {
case (r1: SemanticCheckResult, o: A) =>
(r1, o) =>
val r2 = check(o)(r1.state)
SemanticCheckResult(r2.state, r1.errors ++ r2.errors)
}
Expand Down
Expand Up @@ -48,7 +48,9 @@ class ASTAnnotationMap[K <: ASTNode, V] private (store: Map[(K, InputPosition),
new ASTAnnotationMap(Eagerly.immutableMapValues(store, f))

def replaceKeys(replacements: (K, K)*) = {
val expandedReplacements = replacements.map { case (oldKey, newKey) => (oldKey, oldKey.position) -> (newKey,newKey.position) }
val expandedReplacements = replacements.map {
case (oldKey, newKey) => (oldKey -> oldKey.position) -> (newKey -> newKey.position)
}
val newStore = Eagerly.immutableReplaceKeys(store)(expandedReplacements: _*)
new ASTAnnotationMap(newStore)
}
Expand Down
Expand Up @@ -34,13 +34,13 @@ trait ExecutionPlanInProgressRewriter {
def rewrite(in: ExecutionPlanInProgress)(implicit context: PipeMonitor): ExecutionPlanInProgress
}

class LegacyPipeBuilder(monitors: Monitors, eagernessRewriter: Pipe => Pipe = addEagernessIfNecessary)
class LegacyPipeBuilder(monitors: Monitors, rewriterSequencer: (String) => RewriterStepSequencer, eagernessRewriter: Pipe => Pipe = addEagernessIfNecessary)
extends PatternGraphBuilder with PipeBuilder with GraphQueryBuilder {

private implicit val pipeMonitor: PipeMonitor = monitors.newMonitor[PipeMonitor]()

def producePlan(in: PreparedQuery, planContext: PlanContext): PipeInfo = {
val rewriter = RewriterStepSequencer.newDefault("LegacyPipeBuilder")(reattachAliasedExpressions).rewriter
val rewriter = rewriterSequencer("LegacyPipeBuilder")(reattachAliasedExpressions).rewriter
val rewrite = in.rewrite(rewriter)

rewrite.abstractQuery match {
Expand Down
Expand Up @@ -115,7 +115,7 @@ class TraversalMatcherBuilder extends PlanBuilder with PatternGraphBuilder {
def identifier2nodeFn(ctx:PlanContext, identifier: String, unsolvedItems: Seq[QueryToken[StartItem]]):
(QueryToken[StartItem], EntityProducer[Node]) = {
val startItemQueryToken = unsolvedItems.filter { (item) => identifier == item.token.identifierName }.head
(startItemQueryToken, mapNodeStartCreator()(ctx, startItemQueryToken.token))
(startItemQueryToken, mapNodeStartCreator()((ctx, startItemQueryToken.token)))
}

val entityFactory = new EntityProducerFactory
Expand Down
Expand Up @@ -73,6 +73,9 @@ abstract class TreeZipper[E <: TreeElem[E] : ClassTag] {

case TreeContext(head :: tail, parent, right) =>
Some(Location(head, TreeContext(tail, parent, elem +: right)))

case _ =>
throw new IllegalStateException("Not in tree context when going left")
}

def leftMost: Location = context match {
Expand All @@ -98,6 +101,9 @@ abstract class TreeZipper[E <: TreeElem[E] : ClassTag] {

case TreeContext(left, parent, head :: tail) =>
Some(Location(head, TreeContext(elem +: left, parent, tail)))

case _ =>
throw new IllegalStateException("Not in tree context when going left")
}

def rightMost: Location = context match {
Expand Down
Expand Up @@ -42,12 +42,13 @@ case class CostBasedPipeBuilder(monitors: Monitors,
executionPlanBuilder: PipeExecutionPlanBuilder,
queryPlanner: QueryPlanner,
queryGraphSolver: QueryGraphSolver,
plannerName: CostBasedPlannerName)
plannerName: CostBasedPlannerName,
rewriterSequencer: (String) => RewriterStepSequencer)
extends PipeBuilder {

def producePlan(inputQuery: PreparedQuery, planContext: PlanContext): PipeInfo = {
val statement = CostBasedPipeBuilder.rewriteStatement(inputQuery.statement, inputQuery.scopeTree,
inputQuery.semanticTable, inputQuery.conditions,
inputQuery.semanticTable, rewriterSequencer, inputQuery.conditions,
monitors.newMonitor[AstRewritingMonitor]())
statement match {
case (ast: Query, rewrittenSemanticTable) =>
Expand Down Expand Up @@ -86,17 +87,16 @@ case class CostBasedPipeBuilder(monitors: Monitors,
object CostBasedPipeBuilder {
import org.neo4j.cypher.internal.compiler.v2_2.tracing.rewriters.RewriterStep._

def rewriteStatement(statement: Statement, scopeTree: Scope, semanticTable: SemanticTable, preConditions: Set[RewriterCondition], monitor: AstRewritingMonitor): (Statement, SemanticTable) = {
def rewriteStatement(statement: Statement, scopeTree: Scope, semanticTable: SemanticTable, rewriterSequencer: (String) => RewriterStepSequencer, preConditions: Set[RewriterCondition], monitor: AstRewritingMonitor): (Statement, SemanticTable) = {
val namespacer = Namespacer(statement, scopeTree)
val newStatement = rewriteStatement(namespacer, statement, preConditions, monitor)
val newStatement = rewriteStatement(namespacer, statement, rewriterSequencer, preConditions, monitor)
val newSemanticTable = namespacer.tableRewriter(semanticTable)
(newStatement, newSemanticTable)
}

private def rewriteStatement(namespacer: Namespacer, statement: Statement, preConditions: Set[RewriterCondition], monitor: AstRewritingMonitor): Statement = {
private def rewriteStatement(namespacer: Namespacer, statement: Statement, rewriterSequencer: (String) => RewriterStepSequencer, preConditions: Set[RewriterCondition], monitor: AstRewritingMonitor): Statement = {
val rewriter =
RewriterStepSequencer
.newDefault("Planner")
rewriterSequencer("Planner")
.withPrecondition(preConditions)(
ApplyRewriter("namespaceIdentifiers", namespacer.astRewriter),
rewriteEqualityToInCollection,
Expand Down
Expand Up @@ -24,6 +24,8 @@ import org.neo4j.cypher.internal.compiler.v2_2.planner.execution.PipeExecutionPl
import org.neo4j.cypher.internal.compiler.v2_2.planner.logical._
import org.neo4j.cypher.internal.compiler.v2_2.planner.logical.greedy.{expandsOrJoins, expandsOnly, GreedyQueryGraphSolver}
import org.neo4j.cypher.internal.compiler.v2_2.planner.logical.idp.{IDPQueryGraphSolverMonitor, IDPQueryGraphSolver}
import org.neo4j.cypher.internal.compiler.v2_2.planner.logical.plans.rewriter.LogicalPlanRewriter
import org.neo4j.cypher.internal.compiler.v2_2.tracing.rewriters.RewriterStepSequencer
import org.neo4j.helpers.Clock

object CostBasedPipeBuilderFactory {
Expand All @@ -32,9 +34,10 @@ object CostBasedPipeBuilderFactory {
metricsFactory: MetricsFactory,
monitor: PlanningMonitor,
clock: Clock,
queryPlanner: QueryPlanner,
rewriterSequencer: (String) => RewriterStepSequencer,
tokenResolver: SimpleTokenResolver = new SimpleTokenResolver(),
maybeExecutionPlanBuilder: Option[PipeExecutionPlanBuilder] = None,
queryPlanner: QueryPlanner = new DefaultQueryPlanner(),
plannerName: CostBasedPlannerName = PlannerName.default
) = {

Expand All @@ -56,6 +59,6 @@ object CostBasedPipeBuilderFactory {
)
}

CostBasedPipeBuilder(monitors, metricsFactory, monitor, clock, tokenResolver, executionPlanBuilder, queryPlanner, queryGraphSolver, plannerName)
CostBasedPipeBuilder(monitors, metricsFactory, monitor, clock, tokenResolver, executionPlanBuilder, queryPlanner, queryGraphSolver, plannerName, rewriterSequencer)
}
}
Expand Up @@ -19,8 +19,7 @@
*/
package org.neo4j.cypher.internal.compiler.v2_2.planner.logical

import org.neo4j.cypher.internal.compiler.v2_2.ast.{Collection, HasLabels, Property}
import org.neo4j.cypher.internal.compiler.v2_2.commands.ManyQueryExpression
import org.neo4j.cypher.internal.compiler.v2_2.ast.{HasLabels, Property}
import org.neo4j.cypher.internal.compiler.v2_2.planner.logical.Metrics._
import org.neo4j.cypher.internal.compiler.v2_2.planner.logical.plans._

Expand Down Expand Up @@ -81,9 +80,6 @@ object CardinalityCostModel extends CostModel {
_: NodeIndexScan
=> SLOW_STORE

case NodeIndexSeek(_, _, _, ManyQueryExpression(Collection(elements)), _)
=> SLOW_STORE * Multiplier(elements.size)

case _
=> CPU_BOUND
}
Expand Down Expand Up @@ -119,7 +115,9 @@ object CardinalityCostModel extends CostModel {
case _ =>
val lhsCost = plan.lhs.map(p => apply(p, input)).getOrElse(Cost(0))
val rhsCost = plan.rhs.map(p => apply(p, input)).getOrElse(Cost(0))
val costForThisPlan = cardinalityForPlan(plan) * costPerRow(plan)
val planCardinality = cardinalityForPlan(plan)
val rowCost = costPerRow(plan)
val costForThisPlan = planCardinality * rowCost
val totalCost = costForThisPlan + lhsCost + rhsCost
totalCost
}
Expand Down
Expand Up @@ -24,14 +24,15 @@ import org.neo4j.cypher.internal.compiler.v2_2.planner._
import org.neo4j.cypher.internal.compiler.v2_2.planner.logical.plans.LogicalPlan
import org.neo4j.cypher.internal.compiler.v2_2.planner.logical.plans.rewriter.LogicalPlanRewriter
import org.neo4j.cypher.internal.compiler.v2_2.planner.logical.steps.{aggregation, projection, sortSkipAndLimit, verifyBestPlan}
import org.neo4j.cypher.internal.compiler.v2_2.tracing.rewriters.RewriterStepSequencer

trait QueryPlanner {
def plan(plannerQuery: UnionQuery)(implicit context: LogicalPlanningContext): LogicalPlan
}

class DefaultQueryPlanner(config: QueryPlannerConfiguration = QueryPlannerConfiguration.default,
expressionRewriterFactory: (LogicalPlanningContext => Rewriter) = ExpressionRewriterFactory,
planRewriter: Rewriter = LogicalPlanRewriter)
class DefaultQueryPlanner(planRewriter: Rewriter,
config: QueryPlannerConfiguration = QueryPlannerConfiguration.default,
expressionRewriterFactory: (LogicalPlanningContext => Rewriter) = ExpressionRewriterFactory)
extends QueryPlanner {

def plan(unionQuery: UnionQuery)(implicit context: LogicalPlanningContext): LogicalPlan = unionQuery match {
Expand Down
Expand Up @@ -172,7 +172,7 @@ case class IDPQueryGraphSolver(monitor: IDPQueryGraphSolverMonitor,
oldPlans = Set(lhs, rhs);
newPlan = kit.select(context.logicalPlanProducer.planCartesianProduct(left, right))
)
yield newPlan ->(oldPlans, remaining)).toMap
yield newPlan -> (oldPlans -> remaining)).toMap
val bestCartesian = kit.pickBest(cartesianProducts.keys).get
val (oldPlans, remaining) = cartesianProducts(bestCartesian)
val newPlans = plans.filterNot(oldPlans.contains) :+ (bestCartesian -> remaining)
Expand All @@ -184,7 +184,7 @@ case class IDPQueryGraphSolver(monitor: IDPQueryGraphSolverMonitor,
def applyApplicableOptionalMatches(todo: (LogicalPlan, Seq[QueryGraph])): (/* new plan*/ LogicalPlan, /* remaining */ Seq[QueryGraph]) = {
todo match {
case (plan, allRemaining @ Seq(nextOptional, nextRemaining@_*)) => withOptionalMatch(plan, nextOptional) match {
case Some(newPlan) => applyApplicableOptionalMatches(newPlan, nextRemaining)
case Some(newPlan) => applyApplicableOptionalMatches(newPlan -> nextRemaining)
case None => (plan, allRemaining)
}

Expand Down

0 comments on commit 1f64943

Please sign in to comment.