From 54028ccf62f887753a2b0cf4f3cb0c84889c39dd Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Fri, 5 Aug 2016 14:28:32 +0200 Subject: [PATCH] Implement index usage for subclass hierarchy new SQL executor --- .../sql/executor/OSelectExecutionPlanner.java | 129 +++++++-- .../OSelectStatementExecutionTest.java | 262 +++++++++++++++++- 2 files changed, 361 insertions(+), 30 deletions(-) diff --git a/core/src/main/java/com/orientechnologies/orient/core/sql/executor/OSelectExecutionPlanner.java b/core/src/main/java/com/orientechnologies/orient/core/sql/executor/OSelectExecutionPlanner.java index c83a49d312d..287f86fcfff 100644 --- a/core/src/main/java/com/orientechnologies/orient/core/sql/executor/OSelectExecutionPlanner.java +++ b/core/src/main/java/com/orientechnologies/orient/core/sql/executor/OSelectExecutionPlanner.java @@ -563,8 +563,6 @@ private void handleClassAsTarget(OSelectExecutionPlan plan, OIdentifier identifi return; } - //TODO subclass indexes - Boolean orderByRidAsc = null;//null: no order. true: asc, false:desc if (isOrderByRidAsc()) { orderByRidAsc = true; @@ -576,57 +574,132 @@ private void handleClassAsTarget(OSelectExecutionPlan plan, OIdentifier identifi this.orderApplied = true; } plan.chain(fetcher); + } + + private boolean handleClassAsTargetWithIndex(OSelectExecutionPlan plan, OIdentifier targetClass, OCommandContext ctx) { + List result = handleSubclassAsTargetWithIndex(targetClass.getStringValue(), ctx); + if (result != null) { + result.stream().forEach(x -> plan.chain(x)); + this.whereClause = null; + this.flattenedWhereClause = null; + return true; + } + OClass clazz = ctx.getDatabase().getMetadata().getSchema().getClass(targetClass.getStringValue()); + if (clazz == null) { + throw new OCommandExecutionException("Cannot find class " + targetClass); + } + if (clazz.count(false) != 0 || clazz.getSubclasses().size() == 0 || isDiamondHierarchy(clazz)) { + return false; + } + //try subclasses + + Collection subclasses = clazz.getSubclasses(); + + List subclassPlans = new ArrayList<>(); + for (OClass subClass : subclasses) { + List subSteps = handleSubclassAsTargetWithIndexDeep(subClass.getName(), ctx); + if (subSteps == null || subSteps.size() == 0) { + return false; + } + OSelectExecutionPlan subPlan = new OSelectExecutionPlan(ctx); + subSteps.stream().forEach(x -> subPlan.chain(x)); + subclassPlans.add(subPlan); + } + if (subclassPlans.size() > 0) { + plan.chain(new ParallelExecStep(subclassPlans, ctx)); + return true; + } + return false; + } + + private boolean isDiamondHierarchy(OClass clazz) { + Set traversed = new HashSet<>(); + List stack = new ArrayList<>(); + stack.add(clazz); + while (!stack.isEmpty()) { + OClass current = stack.remove(0); + traversed.add(current); + for (OClass sub : current.getSubclasses()) { + if (traversed.contains(sub)) { + return true; + } + stack.add(sub); + traversed.add(sub); + } + } + return false; + } + + private List handleSubclassAsTargetWithIndexDeep(String targetClass, OCommandContext ctx) { + List result = handleSubclassAsTargetWithIndex(targetClass, ctx); + if (result == null) { + result = new ArrayList<>(); + //TODO recursive on subclassess + OClass clazz = ctx.getDatabase().getMetadata().getSchema().getClass(targetClass); + if (clazz == null) { + throw new OCommandExecutionException("Cannot find class " + targetClass); + } + if (clazz.count(false) != 0 || clazz.getSubclasses().size() == 0 || isDiamondHierarchy(clazz)) { + return null; + } + Collection subclasses = clazz.getSubclasses(); + + List subclassPlans = new ArrayList<>(); + for (OClass subClass : subclasses) { + List subSteps = handleSubclassAsTargetWithIndexDeep(subClass.getName(), ctx); + if (subSteps == null || subSteps.size() == 0) { + return null; + } + OSelectExecutionPlan subPlan = new OSelectExecutionPlan(ctx); + subSteps.stream().forEach(x -> subPlan.chain(x)); + subclassPlans.add(subPlan); + } + if (subclassPlans.size() > 0) { + result.add(new ParallelExecStep(subclassPlans, ctx)); + } + } + return result.size() == 0 ? null : result; } - private boolean handleClassAsTargetWithIndex(OSelectExecutionPlan plan, OIdentifier identifier, OCommandContext ctx) { + private List handleSubclassAsTargetWithIndex(String targetClass, OCommandContext ctx) { if (flattenedWhereClause == null || flattenedWhereClause.size() == 0) { - return false; + return null; } //TODO indexable functions! - OClass clazz = ctx.getDatabase().getMetadata().getSchema().getClass(identifier.getStringValue()); + OClass clazz = ctx.getDatabase().getMetadata().getSchema().getClass(targetClass); + if (clazz == null) { + throw new OCommandExecutionException("Cannot find class " + targetClass); + } + Set> indexes = clazz.getIndexes(); List indexSearchDescriptors = flattenedWhereClause.stream().map(x -> findBestIndexFor(ctx, indexes, x)) .filter(Objects::nonNull).collect(Collectors.toList()); if (indexSearchDescriptors.size() != flattenedWhereClause.size()) { - return false; //some blocks could not be managed with an index + return null; //some blocks could not be managed with an index } + List result = null; List optimumIndexSearchDescriptors = commonFactor(indexSearchDescriptors); if (indexSearchDescriptors.size() == 1) { IndexSearchDescriptor desc = indexSearchDescriptors.get(0); - plan.chain(new FetchFromIndexStep(desc.idx, desc.keyCondition, desc.additionalRangeCondition, ctx)); + result = new ArrayList<>(); + result.add(new FetchFromIndexStep(desc.idx, desc.keyCondition, desc.additionalRangeCondition, ctx)); if (desc.remainingCondition != null && !desc.remainingCondition.isEmpty()) { - plan.chain(new FilterStep(createWhereFrom(desc.remainingCondition), ctx)); + result.add(new FilterStep(createWhereFrom(desc.remainingCondition), ctx)); } - this.whereClause = null; - this.flattenedWhereClause = null; } else { - createParallelIndexFetch(plan, optimumIndexSearchDescriptors, ctx); - this.whereClause = null; - this.flattenedWhereClause = null; + result = new ArrayList<>(); + result.add(createParallelIndexFetch(optimumIndexSearchDescriptors, ctx)); } - return true; - } - - private OExpression toExpression(OCollection right) { - OLevelZeroIdentifier id0 = new OLevelZeroIdentifier(-1); - id0.setCollection(right); - OBaseIdentifier id1 = new OBaseIdentifier(-1); - id1.setLevelZero(id0); - OBaseExpression id2 = new OBaseExpression(-1); - id2.setIdentifier(id1); - OExpression result = new OExpression(-1); - result.setMathExpression(id2); return result; } - private void createParallelIndexFetch(OSelectExecutionPlan plan, List indexSearchDescriptors, - OCommandContext ctx) { + private OExecutionStepInternal createParallelIndexFetch(List indexSearchDescriptors, OCommandContext ctx) { List subPlans = new ArrayList<>(); for (IndexSearchDescriptor desc : indexSearchDescriptors) { OSelectExecutionPlan subPlan = new OSelectExecutionPlan(ctx); @@ -636,7 +709,7 @@ private void createParallelIndexFetch(OSelectExecutionPlan plan, List iSeq = new ArrayList<>(); iSeq.add(i); - iSeq.add(i*2); - iSeq.add(i*4); + iSeq.add(i * 2); + iSeq.add(i * 4); doc.setProperty("iSeq", iSeq); doc.save(); } @@ -1894,6 +1894,264 @@ public class OSelectStatementExecutionTest { result.close(); } + @Test public void testFetchFromSubclassIndexes1() { + String parent = "testFetchFromSubclassIndexes1_parent"; + String child1 = "testFetchFromSubclassIndexes1_child1"; + String child2 = "testFetchFromSubclassIndexes1_child2"; + OClass parentClass = db.getMetadata().getSchema().createClass(parent); + OClass childClass1 = db.getMetadata().getSchema().createClass(child1, parentClass); + OClass childClass2 = db.getMetadata().getSchema().createClass(child2, parentClass); + + parentClass.createProperty("name", OType.STRING); + childClass1.createIndex(child1 + ".name", OClass.INDEX_TYPE.NOTUNIQUE, "name"); + childClass2.createIndex(child2 + ".name", OClass.INDEX_TYPE.NOTUNIQUE, "name"); + + for (int i = 0; i < 10; i++) { + ODocument doc = db.newInstance(child1); + doc.setProperty("name", "name" + i); + doc.save(); + } + + for (int i = 0; i < 10; i++) { + ODocument doc = db.newInstance(child2); + doc.setProperty("name", "name" + i); + doc.save(); + } + + OTodoResultSet result = db.query("select from " + parent + " where name = 'name1'"); + printExecutionPlan(result); + OInternalExecutionPlan plan = (OInternalExecutionPlan) result.getExecutionPlan().get(); + Assert.assertTrue(plan.getSteps().get(0) instanceof ParallelExecStep); + for (int i = 0; i < 2; i++) { + Assert.assertTrue(result.hasNext()); + OResult item = result.next(); + Assert.assertNotNull(item); + } + Assert.assertFalse(result.hasNext()); + result.close(); + } + + @Test public void testFetchFromSubclassIndexes2() { + String parent = "testFetchFromSubclassIndexes2_parent"; + String child1 = "testFetchFromSubclassIndexes2_child1"; + String child2 = "testFetchFromSubclassIndexes2_child2"; + OClass parentClass = db.getMetadata().getSchema().createClass(parent); + OClass childClass1 = db.getMetadata().getSchema().createClass(child1, parentClass); + OClass childClass2 = db.getMetadata().getSchema().createClass(child2, parentClass); + + parentClass.createProperty("name", OType.STRING); + childClass1.createIndex(child1 + ".name", OClass.INDEX_TYPE.NOTUNIQUE, "name"); + childClass2.createIndex(child2 + ".name", OClass.INDEX_TYPE.NOTUNIQUE, "name"); + + for (int i = 0; i < 10; i++) { + ODocument doc = db.newInstance(child1); + doc.setProperty("name", "name" + i); + doc.setProperty("surname", "surname" + i); + doc.save(); + } + + for (int i = 0; i < 10; i++) { + ODocument doc = db.newInstance(child2); + doc.setProperty("name", "name" + i); + doc.setProperty("surname", "surname" + i); + doc.save(); + } + + OTodoResultSet result = db.query("select from " + parent + " where name = 'name1' and surname = 'surname1'"); + printExecutionPlan(result); + OInternalExecutionPlan plan = (OInternalExecutionPlan) result.getExecutionPlan().get(); + Assert.assertTrue(plan.getSteps().get(0) instanceof ParallelExecStep); + for (int i = 0; i < 2; i++) { + Assert.assertTrue(result.hasNext()); + OResult item = result.next(); + Assert.assertNotNull(item); + } + Assert.assertFalse(result.hasNext()); + result.close(); + } + + @Test public void testFetchFromSubclassIndexes3() { + String parent = "testFetchFromSubclassIndexes3_parent"; + String child1 = "testFetchFromSubclassIndexes3_child1"; + String child2 = "testFetchFromSubclassIndexes3_child2"; + OClass parentClass = db.getMetadata().getSchema().createClass(parent); + OClass childClass1 = db.getMetadata().getSchema().createClass(child1, parentClass); + OClass childClass2 = db.getMetadata().getSchema().createClass(child2, parentClass); + + parentClass.createProperty("name", OType.STRING); + childClass1.createIndex(child1 + ".name", OClass.INDEX_TYPE.NOTUNIQUE, "name"); + + for (int i = 0; i < 10; i++) { + ODocument doc = db.newInstance(child1); + doc.setProperty("name", "name" + i); + doc.setProperty("surname", "surname" + i); + doc.save(); + } + + for (int i = 0; i < 10; i++) { + ODocument doc = db.newInstance(child2); + doc.setProperty("name", "name" + i); + doc.setProperty("surname", "surname" + i); + doc.save(); + } + + OTodoResultSet result = db.query("select from " + parent + " where name = 'name1' and surname = 'surname1'"); + printExecutionPlan(result); + OInternalExecutionPlan plan = (OInternalExecutionPlan) result.getExecutionPlan().get(); + Assert.assertTrue(plan.getSteps().get(0) instanceof FetchFromClassExecutionStep);//no index used + for (int i = 0; i < 2; i++) { + Assert.assertTrue(result.hasNext()); + OResult item = result.next(); + Assert.assertNotNull(item); + } + Assert.assertFalse(result.hasNext()); + result.close(); + } + + @Test public void testFetchFromSubclassIndexes4() { + String parent = "testFetchFromSubclassIndexes4_parent"; + String child1 = "testFetchFromSubclassIndexes4_child1"; + String child2 = "testFetchFromSubclassIndexes4_child2"; + OClass parentClass = db.getMetadata().getSchema().createClass(parent); + OClass childClass1 = db.getMetadata().getSchema().createClass(child1, parentClass); + OClass childClass2 = db.getMetadata().getSchema().createClass(child2, parentClass); + + parentClass.createProperty("name", OType.STRING); + childClass1.createIndex(child1 + ".name", OClass.INDEX_TYPE.NOTUNIQUE, "name"); + childClass2.createIndex(child2 + ".name", OClass.INDEX_TYPE.NOTUNIQUE, "name"); + + ODocument parentdoc = db.newInstance(parent); + parentdoc.setProperty("name", "foo"); + parentdoc.save(); + + for (int i = 0; i < 10; i++) { + ODocument doc = db.newInstance(child1); + doc.setProperty("name", "name" + i); + doc.setProperty("surname", "surname" + i); + doc.save(); + } + + for (int i = 0; i < 10; i++) { + ODocument doc = db.newInstance(child2); + doc.setProperty("name", "name" + i); + doc.setProperty("surname", "surname" + i); + doc.save(); + } + + OTodoResultSet result = db.query("select from " + parent + " where name = 'name1' and surname = 'surname1'"); + printExecutionPlan(result); + OInternalExecutionPlan plan = (OInternalExecutionPlan) result.getExecutionPlan().get(); + Assert + .assertTrue(plan.getSteps().get(0) instanceof FetchFromClassExecutionStep); //no index, because the superclass is not empty + for (int i = 0; i < 2; i++) { + Assert.assertTrue(result.hasNext()); + OResult item = result.next(); + Assert.assertNotNull(item); + } + Assert.assertFalse(result.hasNext()); + result.close(); + } + + @Test public void testFetchFromSubSubclassIndexes() { + String parent = "testFetchFromSubSubclassIndexes_parent"; + String child1 = "testFetchFromSubSubclassIndexes_child1"; + String child2 = "testFetchFromSubSubclassIndexes_child2"; + String child2_1 = "testFetchFromSubSubclassIndexes_child2_1"; + String child2_2 = "testFetchFromSubSubclassIndexes_child2_2"; + OClass parentClass = db.getMetadata().getSchema().createClass(parent); + OClass childClass1 = db.getMetadata().getSchema().createClass(child1, parentClass); + OClass childClass2 = db.getMetadata().getSchema().createClass(child2, parentClass); + OClass childClass2_1 = db.getMetadata().getSchema().createClass(child2_1, childClass2); + OClass childClass2_2 = db.getMetadata().getSchema().createClass(child2_2, childClass2); + + parentClass.createProperty("name", OType.STRING); + childClass1.createIndex(child1 + ".name", OClass.INDEX_TYPE.NOTUNIQUE, "name"); + childClass2_1.createIndex(child2_1 + ".name", OClass.INDEX_TYPE.NOTUNIQUE, "name"); + childClass2_2.createIndex(child2_2 + ".name", OClass.INDEX_TYPE.NOTUNIQUE, "name"); + + for (int i = 0; i < 10; i++) { + ODocument doc = db.newInstance(child1); + doc.setProperty("name", "name" + i); + doc.setProperty("surname", "surname" + i); + doc.save(); + } + + for (int i = 0; i < 10; i++) { + ODocument doc = db.newInstance(child2_1); + doc.setProperty("name", "name" + i); + doc.setProperty("surname", "surname" + i); + doc.save(); + } + + for (int i = 0; i < 10; i++) { + ODocument doc = db.newInstance(child2_2); + doc.setProperty("name", "name" + i); + doc.setProperty("surname", "surname" + i); + doc.save(); + } + + OTodoResultSet result = db.query("select from " + parent + " where name = 'name1' and surname = 'surname1'"); + printExecutionPlan(result); + OInternalExecutionPlan plan = (OInternalExecutionPlan) result.getExecutionPlan().get(); + Assert.assertTrue(plan.getSteps().get(0) instanceof ParallelExecStep); + for (int i = 0; i < 3; i++) { + Assert.assertTrue(result.hasNext()); + OResult item = result.next(); + Assert.assertNotNull(item); + } + Assert.assertFalse(result.hasNext()); + result.close(); + } + + @Test public void testFetchFromSubSubclassIndexesWithDiamond() { + String parent = "testFetchFromSubSubclassIndexesWithDiamond_parent"; + String child1 = "testFetchFromSubSubclassIndexesWithDiamond_child1"; + String child2 = "testFetchFromSubSubclassIndexesWithDiamond_child2"; + String child12 = "testFetchFromSubSubclassIndexesWithDiamond_child12"; + + OClass parentClass = db.getMetadata().getSchema().createClass(parent); + OClass childClass1 = db.getMetadata().getSchema().createClass(child1, parentClass); + OClass childClass2 = db.getMetadata().getSchema().createClass(child2, parentClass); + OClass childClass12 = db.getMetadata().getSchema().createClass(child12, childClass1, childClass2); + + parentClass.createProperty("name", OType.STRING); + childClass1.createIndex(child1 + ".name", OClass.INDEX_TYPE.NOTUNIQUE, "name"); + childClass2.createIndex(child2 + ".name", OClass.INDEX_TYPE.NOTUNIQUE, "name"); + + for (int i = 0; i < 10; i++) { + ODocument doc = db.newInstance(child1); + doc.setProperty("name", "name" + i); + doc.setProperty("surname", "surname" + i); + doc.save(); + } + + for (int i = 0; i < 10; i++) { + ODocument doc = db.newInstance(child2); + doc.setProperty("name", "name" + i); + doc.setProperty("surname", "surname" + i); + doc.save(); + } + + for (int i = 0; i < 10; i++) { + ODocument doc = db.newInstance(child12); + doc.setProperty("name", "name" + i); + doc.setProperty("surname", "surname" + i); + doc.save(); + } + + OTodoResultSet result = db.query("select from " + parent + " where name = 'name1' and surname = 'surname1'"); + printExecutionPlan(result); + OInternalExecutionPlan plan = (OInternalExecutionPlan) result.getExecutionPlan().get(); + Assert.assertTrue(plan.getSteps().get(0) instanceof FetchFromClassExecutionStep); + for (int i = 0; i < 3; i++) { + Assert.assertTrue(result.hasNext()); + OResult item = result.next(); + Assert.assertNotNull(item); + } + Assert.assertFalse(result.hasNext()); + result.close(); + } + public void stressTestNew() { String className = "stressTestNew"; db.getMetadata().getSchema().createClass(className);