Skip to content

Commit

Permalink
Add DbStructureLogicalPlanningConfiguration
Browse files Browse the repository at this point in the history
  • Loading branch information
boggle committed Feb 22, 2015
1 parent 0aa139f commit bef05d6
Show file tree
Hide file tree
Showing 17 changed files with 824 additions and 140 deletions.
Expand Up @@ -26,3 +26,10 @@ sealed trait NameId {
final case class LabelId(id: Int) extends NameId
final case class RelTypeId(id: Int) extends NameId
final case class PropertyKeyId(id: Int) extends NameId

object NameId {
val WILDCARD: Int = -1

implicit def toKernelEncode(nameId: NameId): Int = nameId.id
implicit def toKernelEncode(nameId: Option[NameId]): Int = nameId.map(toKernelEncode).getOrElse(WILDCARD)
}
Expand Up @@ -22,6 +22,14 @@ package org.neo4j.cypher.internal.compiler.v2_2.spi
import org.neo4j.cypher.internal.compiler.v2_2.planner.logical.{Cardinality, Selectivity}
import org.neo4j.cypher.internal.compiler.v2_2.{LabelId, PropertyKeyId, RelTypeId}

object GraphStatistics {
val DEFAULT_RANGE_SELECTIVITY = Selectivity(0.3)
val DEFAULT_PREDICATE_SELECTIVITY = Selectivity(0.75)
val DEFAULT_EQUALITY_SELECTIVITY = Selectivity(0.1)
val DEFAULT_NUMBER_OF_ID_LOOKUPS = Cardinality(25)
val DEFAULT_REL_UNIQUENESS_SELECTIVITY = Selectivity(1.0 - 1 / 100 /*rel-cardinality*/)
}

trait GraphStatistics {
def nodesWithLabelCardinality(labelId: Option[LabelId]): Cardinality

Expand All @@ -35,10 +43,35 @@ trait GraphStatistics {
def indexSelectivity(label: LabelId, property: PropertyKeyId): Option[Selectivity]
}

object GraphStatistics {
val DEFAULT_RANGE_SELECTIVITY = Selectivity(0.3)
val DEFAULT_PREDICATE_SELECTIVITY = Selectivity(0.75)
val DEFAULT_EQUALITY_SELECTIVITY = Selectivity(0.1)
val DEFAULT_NUMBER_OF_ID_LOOKUPS = Cardinality(25)
val DEFAULT_REL_UNIQUENESS_SELECTIVITY = Selectivity(1.0 - 1 / 100 /*rel-cardinality*/)
class DelegatingGraphStatistics(delegate: GraphStatistics) extends GraphStatistics {
override def nodesWithLabelCardinality(labelId: Option[LabelId]): Cardinality =
delegate.nodesWithLabelCardinality(labelId)

override def cardinalityByLabelsAndRelationshipType(fromLabel: Option[LabelId], relTypeId: Option[RelTypeId], toLabel: Option[LabelId]): Cardinality =
delegate.cardinalityByLabelsAndRelationshipType(fromLabel, relTypeId, toLabel)

/*
Probability of any node with the given label, to have a property with a given value
indexSelectivity(:X, prop) = s => |MATCH (a:X)| * s = |MATCH (a:X) WHERE x.prop = *|
*/
override def indexSelectivity(label: LabelId, property: PropertyKeyId): Option[Selectivity] =
delegate.indexSelectivity(label, property)
}

class StatisticsCompletingGraphStatistics(delegate: GraphStatistics)
extends DelegatingGraphStatistics(delegate) {

override def cardinalityByLabelsAndRelationshipType(fromLabel: Option[LabelId], relTypeId: Option[RelTypeId], toLabel: Option[LabelId]): Cardinality =
(fromLabel, toLabel) match {
case (Some(_), Some(_)) =>
// TODO: read real counts from readOperations when they are gonna be properly computed and updated
Cardinality.min(
super.cardinalityByLabelsAndRelationshipType(fromLabel, relTypeId, None),
super.cardinalityByLabelsAndRelationshipType(None, relTypeId, toLabel)
)
case _ =>
super.cardinalityByLabelsAndRelationshipType(fromLabel, relTypeId, toLabel)
}
}

@@ -0,0 +1,46 @@
/**
* Copyright (c) 2002-2015 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.cypher.internal.compiler.v2_2.planner

import org.neo4j.cypher.internal.compiler.v2_2.{NameId, PropertyKeyId, RelTypeId, LabelId}
import org.neo4j.cypher.internal.compiler.v2_2.planner.logical.{Selectivity, Cardinality}
import org.neo4j.cypher.internal.compiler.v2_2.spi.GraphStatistics
import org.neo4j.kernel.impl.util.dbstructure.DbStructureLookup

class DbStructureGraphStatistics(lookup: DbStructureLookup) extends GraphStatistics {

import NameId._

override def nodesWithLabelCardinality( label: Option[LabelId] ): Cardinality =
Cardinality(lookup.nodesWithLabelCardinality(label))

override def cardinalityByLabelsAndRelationshipType( fromLabel: Option[LabelId], relTypeId: Option[RelTypeId], toLabel: Option[LabelId] ): Cardinality =
Cardinality(lookup.cardinalityByLabelsAndRelationshipType(fromLabel, relTypeId, toLabel))

/*
Probability of any node with the given label, to have a property with a given value
indexSelectivity(:X, prop) = s => |MATCH (a:X)| * s = |MATCH (a:X) WHERE x.prop = *|
*/
override def indexSelectivity( label: LabelId, property: PropertyKeyId ): Option[Selectivity] = {
val result = lookup.indexSelectivity( label.id, property.id )
if (result.isNaN) None else Some(Selectivity(result))
}
}
@@ -0,0 +1,75 @@
/**
* Copyright (c) 2002-2015 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.cypher.internal.compiler.v2_2.planner

import java.util

import org.neo4j.cypher.internal.compiler.v2_2.spi.{GraphStatistics, StatisticsCompletingGraphStatistics}
import org.neo4j.cypher.internal.compiler.v2_2.{LabelId, PropertyKeyId, RelTypeId}
import org.neo4j.helpers.Pair
import org.neo4j.helpers.collection.Visitable
import org.neo4j.kernel.impl.util.dbstructure.{DbStructureCollector, DbStructureLookup, DbStructureVisitor}

import scala.collection.JavaConverters._
import scala.collection.mutable

object DbStructureLogicalPlanningConfiguration {

def apply(visitable: Visitable[DbStructureVisitor]): LogicalPlanningConfiguration = {
val collector = new DbStructureCollector
visitable.accept(collector)
val lookup = collector.lookup()
apply(lookup, new DbStructureGraphStatistics(lookup))
}

def apply(lookup: DbStructureLookup, underlyingStatistics: GraphStatistics): LogicalPlanningConfiguration = {
val resolvedLabels: mutable.Map[String, LabelId] = resolveTokens(lookup.labels())(LabelId)
val resolvedPropertyKeys = resolveTokens(lookup.properties())(PropertyKeyId)
val resolvedRelTypeNames = resolveTokens(lookup.relationshipTypes())(RelTypeId)

new RealLogicalPlanningConfiguration {

override val computeSemanticTable: SemanticTable = new SemanticTable(
resolvedLabelIds = resolvedLabels,
resolvedPropertyKeyNames = resolvedPropertyKeys,
resolvedRelTypeNames = resolvedRelTypeNames
)

override val graphStatistics: GraphStatistics =
new StatisticsCompletingGraphStatistics(underlyingStatistics)

override val indexes: Set[(String, String)] = indexSet(lookup.knownIndices())
override val knownLabels: Set[String] = resolvedLabels.keys.toSet
override val uniqueIndexes: Set[(String, String)] = indexSet(lookup.knownUniqueIndices())
}
}

private def indexSet(indices: util.Iterator[Pair[String, String]]): Set[(String, String)] =
indices.asScala.map { pair => pair.first() -> pair.other() }.toSet

private def resolveTokens[T](iterator: util.Iterator[Pair[Integer, String]])(f: Int => T): mutable.Map[String, T] = {
val builder = mutable.Map.newBuilder[String, T]
while (iterator.hasNext) {
val next = iterator.next()
builder += next.other() -> f(next.first())
}
builder.result()
}
}
Expand Up @@ -19,18 +19,11 @@
*/
package org.neo4j.cypher.internal.compiler.v2_2.planner

import org.neo4j.cypher.internal.compiler.v2_2.{RelTypeId, PropertyKeyId, LabelId, HardcodedGraphStatistics}
import org.neo4j.cypher.internal.compiler.v2_2.ast.Expression
import org.neo4j.cypher.internal.compiler.v2_2._
import org.neo4j.cypher.internal.compiler.v2_2.planner.logical.Metrics._
import org.neo4j.cypher.internal.compiler.v2_2.planner.logical._
import org.neo4j.cypher.internal.compiler.v2_2.planner.logical.plans.{NodeByLabelScan, LogicalPlan}
import org.neo4j.cypher.internal.compiler.v2_2.planner.logical.plans.LogicalPlan
import org.neo4j.cypher.internal.compiler.v2_2.planner.logical.{Cost, _}
import org.neo4j.cypher.internal.compiler.v2_2.spi.GraphStatistics
import org.neo4j.helpers.collection.Visitable
import org.neo4j.kernel.api.constraints.UniquenessConstraint
import org.neo4j.kernel.api.index.IndexDescriptor
import org.neo4j.kernel.impl.util.dbstructure.DbStructureVisitor

import scala.collection.mutable

trait LogicalPlanningConfiguration {
def computeSemanticTable: SemanticTable
Expand Down Expand Up @@ -68,72 +61,3 @@ trait LogicalPlanningConfigurationAdHocSemanticTable {
table
}
}
case class RealLogicalPlanningConfiguration()
extends LogicalPlanningConfiguration with LogicalPlanningConfigurationAdHocSemanticTable {

def cardinalityModel(queryGraphCardinalityModel: QueryGraphCardinalityModel, semanticTable: SemanticTable) = {
val model: Metrics.CardinalityModel = new StatisticsBackedCardinalityModel(queryGraphCardinalityModel)
({ case (plan: LogicalPlan, card: QueryGraphCardinalityInput) => model(plan, card) })
}

def costModel(cardinality: CardinalityModel): PartialFunction[LogicalPlan, Cost] = {
val model: Metrics.CostModel = new CardinalityCostModel(cardinality)
({ case (plan: LogicalPlan) => model(plan, QueryGraphCardinalityInput.empty) })
}

def graphStatistics: GraphStatistics = HardcodedGraphStatistics

def indexes = Set.empty
def uniqueIndexes = Set.empty
def labelCardinality = Map.empty
def knownLabels = Set.empty

def qg: QueryGraph = ???
}

class StubbedLogicalPlanningConfiguration(parent: LogicalPlanningConfiguration)
extends LogicalPlanningConfiguration with LogicalPlanningConfigurationAdHocSemanticTable {

self =>

var knownLabels: Set[String] = Set.empty
var cardinality: PartialFunction[LogicalPlan, Cardinality] = PartialFunction.empty
var cost: PartialFunction[LogicalPlan, Cost] = PartialFunction.empty
var selectivity: PartialFunction[Expression, Selectivity] = PartialFunction.empty
var labelCardinality: Map[String, Cardinality] = Map.empty
var statistics = null
var qg: QueryGraph = null

var indexes: Set[(String, String)] = Set.empty
var uniqueIndexes: Set[(String, String)] = Set.empty
def indexOn(label: String, property: String) {
indexes = indexes + (label -> property)
}
def uniqueIndexOn(label: String, property: String) {
uniqueIndexes = uniqueIndexes + (label -> property)
}

def costModel(cardinality: Metrics.CardinalityModel) =
cost.orElse(parent.costModel(cardinality))

def cardinalityModel(queryGraphCardinalityModel: QueryGraphCardinalityModel, semanticTable: SemanticTable): Metrics.CardinalityModel = {
val labelIdCardinality: Map[LabelId, Cardinality] = labelCardinality.map {
case (name: String, cardinality: Cardinality) =>
semanticTable.resolvedLabelIds(name) -> cardinality
}
val labelScanCardinality: PartialFunction[LogicalPlan, Cardinality] = {
case NodeByLabelScan(_, label, _) if label.id(semanticTable).isDefined &&
labelIdCardinality.contains(label.id(semanticTable).get) =>
labelIdCardinality(label.id(semanticTable).get)
}

val r: PartialFunction[LogicalPlan, Cardinality] =
labelScanCardinality.orElse(cardinality)

(p: LogicalPlan, c: QueryGraphCardinalityInput) =>
if(r.isDefinedAt(p)) r.apply(p) else parent.cardinalityModel(queryGraphCardinalityModel, semanticTable)(p, c)
}

def graphStatistics: GraphStatistics =
Option(statistics).getOrElse(parent.graphStatistics)
}
Expand Up @@ -91,6 +91,16 @@ trait LogicalPlanningTestSupport2 extends CypherTestSupport with AstConstruction
else
None

def getUniquenessConstraint(labelName: String, propertyKey: String): Option[UniquenessConstraint] = {
if (config.uniqueIndexes((labelName, propertyKey)))
Some(new UniquenessConstraint(
semanticTable.resolvedLabelIds(labelName).id,
semanticTable.resolvedPropertyKeyNames(propertyKey).id
))
else
None
}

def getIndexRule(labelName: String, propertyKey: String): Option[IndexDescriptor] =
if (config.indexes((labelName, propertyKey)))
Some(new IndexDescriptor(
Expand All @@ -110,7 +120,6 @@ trait LogicalPlanningTestSupport2 extends CypherTestSupport with AstConstruction
semanticTable.resolvedRelTypeNames.get(relType).map(_.id)

def checkNodeIndex(idxName: String): Unit = ???
def getUniquenessConstraint(labelName: String, propertyKey: String): Option[UniquenessConstraint] = ???
def checkRelIndex(idxName: String): Unit = ???
def getOrCreateFromSchemaState[T](key: Any, f: => T): T = ???
def getRelTypeName(id: Int): String = ???
Expand Down
@@ -0,0 +1,52 @@
/**
* Copyright (c) 2002-2015 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.cypher.internal.compiler.v2_2.planner

import org.neo4j.cypher.internal.compiler.v2_2.HardcodedGraphStatistics
import org.neo4j.cypher.internal.compiler.v2_2.planner.logical.plans.LogicalPlan
import org.neo4j.cypher.internal.compiler.v2_2.planner.logical.{CardinalityCostModel, Cost, StatisticsBackedCardinalityModel, Metrics}
import org.neo4j.cypher.internal.compiler.v2_2.planner.logical.Metrics.{CardinalityModel, QueryGraphCardinalityInput, QueryGraphCardinalityModel}
import org.neo4j.cypher.internal.compiler.v2_2.spi.GraphStatistics

case class RealLogicalPlanningConfiguration()
extends LogicalPlanningConfiguration with LogicalPlanningConfigurationAdHocSemanticTable {

def cardinalityModel(queryGraphCardinalityModel: QueryGraphCardinalityModel, semanticTable: SemanticTable) = {
val model: Metrics.CardinalityModel = new StatisticsBackedCardinalityModel(queryGraphCardinalityModel)
({
case (plan: LogicalPlan, card: QueryGraphCardinalityInput) => model(plan, card)
})
}

def costModel(cardinality: CardinalityModel): PartialFunction[LogicalPlan, Cost] = {
val model: Metrics.CostModel = new CardinalityCostModel(cardinality)
({
case (plan: LogicalPlan) => model(plan, QueryGraphCardinalityInput.empty)
})
}

def graphStatistics: GraphStatistics = HardcodedGraphStatistics
def indexes = Set.empty
def uniqueIndexes = Set.empty
def labelCardinality = Map.empty
def knownLabels = Set.empty

def qg: QueryGraph = ???
}

0 comments on commit bef05d6

Please sign in to comment.