From 967a4b3d5237ebd1e3d72af497a7d8e8bb8cb129 Mon Sep 17 00:00:00 2001 From: devozerov Date: Mon, 10 Dec 2018 15:25:27 +0300 Subject: [PATCH 01/80] Experimenting. --- .../query/QueryRewritePlaygroundTest.java | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/QueryRewritePlaygroundTest.java diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/QueryRewritePlaygroundTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/QueryRewritePlaygroundTest.java new file mode 100644 index 0000000000000..555f6f9a790cf --- /dev/null +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/QueryRewritePlaygroundTest.java @@ -0,0 +1,64 @@ +package org.apache.ignite.internal.processors.query; + +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; + +public class QueryRewritePlaygroundTest extends GridCommonAbstractTest { + /** IP finder. */ + private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + startGrid(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER)); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected long getTestTimeout() { + return Long.MAX_VALUE; + } + + /** + * @throws Exception If failed. + */ + public void testVarious() throws Exception { + executeSql("CREATE TABLE dept (id BIGINT PRIMARY KEY, name VARCHAR)"); + executeSql("CREATE TABLE emp (id BIGINT PRIMARY KEY, name VARCHAR, dept_id BIGINT, salary BIGINT)"); + + //executeSql("SELECT emp.name, dept.name FROM emp, dept WHERE emp.dept_id=dept.id"); + executeSql("SELECT emp.name, dept.name FROM emp INNER JOIN dept ON emp.dept_id=dept.id"); + +// executeSql("" + +// "SELECT emp.name, (SELECT dept.name FROM dept WHERE emp.dept_id=dept.id)\n" + +// "FROM emp\n" + +// "WHERE emp.salary > 1000" +// ); + } + + public static void executeSql(String sql) throws Exception { + try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1")) { + try (Statement stmt = conn.createStatement()) { + stmt.execute(sql); + } + } + } +} From 1420cf54e81b6450b7fa5929d96f75093b9652e9 Mon Sep 17 00:00:00 2001 From: devozerov Date: Tue, 11 Dec 2018 15:18:51 +0300 Subject: [PATCH 02/80] Initial modelling. --- .../join/PartitionJoinAffinityIdentifier.java | 28 +++++++ .../affinity/join/PartitionJoinCondition.java | 78 +++++++++++++++++++ .../h2/affinity/join/PartitionJoinGroup.java | 24 ++++++ .../h2/affinity/join/PartitionJoinModel.java | 24 ++++++ .../h2/affinity/join/PartitionJoinTable.java | 56 +++++++++++++ 5 files changed, 210 insertions(+) create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinAffinityIdentifier.java create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinCondition.java create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinModel.java create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinAffinityIdentifier.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinAffinityIdentifier.java new file mode 100644 index 0000000000000..1dc2439c90177 --- /dev/null +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinAffinityIdentifier.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.h2.affinity.join; + +import java.io.Serializable; + +/** + * Affinity function identifier. Used to compare affinity functions of two tables. + */ +public interface PartitionJoinAffinityIdentifier extends Serializable { + + +} diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinCondition.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinCondition.java new file mode 100644 index 0000000000000..5dc51e285d489 --- /dev/null +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinCondition.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.h2.affinity.join; + +/** + * Join condition. + */ +public class PartitionJoinCondition { + /** Left alias. */ + private final String leftAlias; + + /** Right alias. */ + private final String rightAlias; + + /** Left column name. */ + private final String leftCol; + + /** Right column name. */ + private final String rightCol; + + /** + * Constructor. + * + * @param leftAlias Left alias. + * @param rightAlias Right alias. + * @param leftCol Left column name. + * @param rightCol Right column name. + */ + public PartitionJoinCondition(String leftAlias, String rightAlias, String leftCol, String rightCol) { + this.leftAlias = leftAlias; + this.rightAlias = rightAlias; + this.leftCol = leftCol; + this.rightCol = rightCol; + } + + /** + * Left alias. + */ + public String leftAlias() { + return leftAlias; + } + + /** + * Right alias. + */ + public String rightAlias() { + return rightAlias; + } + + /** + * @return Left column. + */ + public String leftColumn() { + return leftCol; + } + + /** + * @return Right column. + */ + public String rightColumn() { + return rightCol; + } +} diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java new file mode 100644 index 0000000000000..ee6bcdecbf63a --- /dev/null +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java @@ -0,0 +1,24 @@ +package org.apache.ignite.internal.processors.query.h2.affinity.join; + +import java.util.Collection; +import java.util.Collections; +import java.util.IdentityHashMap; + +public class PartitionJoinGroup { + /** Tables within a group. */ + private final Collection tbls = Collections.newSetFromMap(new IdentityHashMap<>()); + + /** Tables that were left joined to the group (i.e. these are tables that were on the right side of LJ. */ + private final Collection outerTbls = Collections.newSetFromMap(new IdentityHashMap<>()); + + /** Affinity function identifier. */ + private final PartitionJoinAffinityIdentifier affIdentifier; + + /** Whether this is replicated group. */ + private final boolean replicated; + + public PartitionJoinGroup(PartitionJoinAffinityIdentifier affIdentifier, boolean replicated) { + this.affIdentifier = affIdentifier; + this.replicated = replicated; + } +} diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinModel.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinModel.java new file mode 100644 index 0000000000000..eb06209610c0a --- /dev/null +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinModel.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.h2.affinity.join; + +/** + * Partition join model. Describes how tables are joined with each other. + */ +public class PartitionJoinModel { +} diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java new file mode 100644 index 0000000000000..2f5b78c0f6477 --- /dev/null +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.h2.affinity.join; + +/** + * Single table with affinity info. + */ +public class PartitionJoinTable { + /** Alias used in the query. */ + private final String alias; + + /** Cache name. */ + private final String cacheName; + + /** Affinity column name (if can be resolved). */ + private final String affColName; + + /** Whether table is left joined. */ + private boolean leftJoined; + + public PartitionJoinTable( + String alias, + String cacheName, + PartitionJoinAffinityIdentifier affIdentifier, + String affColName, + boolean replicated, + boolean subquery + ) { + this.alias = alias; + this.cacheName = cacheName; + this.affColName = affColName; + } + + public void leftJoined(boolean leftJoined) { + this.leftJoined = leftJoined; + } + + public boolean leftJoined() { + return leftJoined; + } +} From bffb5888ec7cf99d52b0319cd887b4d03ccafde8 Mon Sep 17 00:00:00 2001 From: devozerov Date: Tue, 11 Dec 2018 16:36:44 +0300 Subject: [PATCH 03/80] WIP. --- .../query/h2/affinity/PartitionExtractor.java | 107 ++++++++++++++++-- .../affinity/join/PartitionJoinCondition.java | 40 ++++++- .../h2/affinity/join/PartitionJoinGroup.java | 41 ++++++- 3 files changed, 176 insertions(+), 12 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java index c57b011919967..2ed2d43ae9267 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java @@ -20,6 +20,9 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery; import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing; +import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinCondition; +import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinGroup; +import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinModel; import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor; import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlAlias; @@ -27,6 +30,7 @@ import org.apache.ignite.internal.processors.query.h2.sql.GridSqlColumn; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlConst; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlElement; +import org.apache.ignite.internal.processors.query.h2.sql.GridSqlJoin; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlOperation; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlOperationType; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlParameter; @@ -38,8 +42,11 @@ import org.h2.table.IndexColumn; import org.jetbrains.annotations.Nullable; +import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import static org.apache.ignite.internal.processors.query.h2.opt.GridH2KeyValueRowOnheap.DEFAULT_COLUMNS_COUNT; @@ -73,6 +80,8 @@ public PartitionResult extract(GridSqlQuery qry) throws IgniteCheckedException { GridSqlSelect select = (GridSqlSelect)qry; + prepareJoinModel(select.from(), select.where()); + // Currently we can extract data only from a single table. GridSqlTable tbl = unwrapTable(select.from()); @@ -143,13 +152,82 @@ else if (!F.eq(desc, qryRes.descriptor())) return new PartitionResult(desc, tree); } + /** + * Prepare join model. + * + * @param from FROM clause. + * @param where WHERE clause. + * @return Join model. + */ + private PartitionJoinModel prepareJoinModel(GridSqlAst from, GridSqlAst where) { + Map grps = new HashMap<>(); + Collection conds = new HashSet<>(); + + prepareJoinModelTables(from, grps, conds); + + // TODO + return null; + } + + /** + * Prepare tables which will be used in join model. + * + * @param from From flag. + * @param grps Groups. + * @param conds Conditions. + * @return {@code True} if extracted tables successfully, {@code false} if failed to extract. + */ + private boolean prepareJoinModelTables( + GridSqlAst from, + Map grps, + Collection conds + ) { + if (from instanceof GridSqlJoin) { + // Process JOIN recursively. + GridSqlJoin join = (GridSqlJoin)from; + + if (!prepareJoinModelTables(join.leftTable(), grps, conds)) + return false; + + if (!prepareJoinModelTables(join.rightTable(), grps, conds)) + return false; + + // Make sure that ON condition is simple equality. Stop process otherwise. + GridSqlElement on = join.on(); + + boolean onSimple = false; + + if (on instanceof GridSqlOperation) { + GridSqlOperation on0 = (GridSqlOperation)on; + + if (on0.operationType() == GridSqlOperationType.EQUAL) { + GridSqlColumn left = unwrapColumn(on0.child(0)); + GridSqlColumn right = unwrapColumn(on0.child(1)); + + // TODO + System.out.println(left + " " + right); + } + } + } + + String alias = null; + + if (from instanceof GridSqlAlias) { + alias = ((GridSqlAlias)from).alias(); + + from = from.child(); + } + + return true; + } + /** * Try unwrapping the table. * * @param from From. * @return Table or {@code null} if not a table. */ - @Nullable private static GridSqlTable unwrapTable(GridSqlAst from) { + @Nullable private static GridSqlTable unwrapTable(GridSqlAst from) { if (from instanceof GridSqlAlias) from = from.child(); @@ -240,11 +318,9 @@ private PartitionNode extractFromIn(GridSqlOperation op) throws IgniteCheckedExc // Left operand should be column. GridSqlAst left = op.child(); - GridSqlColumn leftCol; + GridSqlColumn leftCol = unwrapColumn(left); - if (left instanceof GridSqlColumn) - leftCol = (GridSqlColumn)left; - else + if (leftCol == null) return PartitionAllNode.INSTANCE; // Can work only with Ignite tables. @@ -273,7 +349,7 @@ else if (right instanceof GridSqlParameter) { // set globally. Hence, returning null. return PartitionAllNode.INSTANCE; - // Do extract. + // Extract. PartitionSingleNode part = extractSingle(leftCol.column(), rightConst, rightParam); // Same thing as above: single unknown partition in disjunction defeats optimization. @@ -298,11 +374,9 @@ private PartitionNode extractFromEqual(GridSqlOperation op) throws IgniteChecked GridSqlElement left = op.child(0); GridSqlElement right = op.child(1); - GridSqlColumn leftCol; + GridSqlColumn leftCol = unwrapColumn(left); - if (left instanceof GridSqlColumn) - leftCol = (GridSqlColumn)left; - else + if (leftCol == null) return PartitionAllNode.INSTANCE; if (!(leftCol.column().getTable() instanceof GridH2Table)) @@ -394,4 +468,17 @@ private static boolean isAffinityKey(int colId, GridH2Table tbl) { private static PartitionTableDescriptor descriptor(GridH2Table tbl) { return new PartitionTableDescriptor(tbl.cacheName(), tbl.getName()); } + + /** + * Unwrap column if possible. + * + * @param ast AST. + * @return Column or {@code null} if not a column. + */ + @Nullable private static GridSqlColumn unwrapColumn(GridSqlAst ast) { + if (ast instanceof GridSqlAlias) + ast = ast.child(); + + return ast instanceof GridSqlColumn ? (GridSqlColumn)ast : null; + } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinCondition.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinCondition.java index 5dc51e285d489..b2d3f0d347692 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinCondition.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinCondition.java @@ -17,6 +17,8 @@ package org.apache.ignite.internal.processors.query.h2.affinity.join; +import org.apache.ignite.internal.util.typedef.F; + /** * Join condition. */ @@ -33,6 +35,9 @@ public class PartitionJoinCondition { /** Right column name. */ private final String rightCol; + /** Whether this is LEFT OUTER JOIN. */ + private final boolean left; + /** * Constructor. * @@ -40,12 +45,14 @@ public class PartitionJoinCondition { * @param rightAlias Right alias. * @param leftCol Left column name. * @param rightCol Right column name. + * @param left Left join flag. */ - public PartitionJoinCondition(String leftAlias, String rightAlias, String leftCol, String rightCol) { + public PartitionJoinCondition(String leftAlias, String rightAlias, String leftCol, String rightCol, boolean left) { this.leftAlias = leftAlias; this.rightAlias = rightAlias; this.leftCol = leftCol; this.rightCol = rightCol; + this.left = left; } /** @@ -75,4 +82,35 @@ public String leftColumn() { public String rightColumn() { return rightCol; } + + /** + * @return Whether this is left join. + */ + public boolean left() { + return left; + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + int res = leftAlias.hashCode(); + + res = 31 * res + rightAlias.hashCode(); + res = 31 * res + leftCol.hashCode(); + res = 31 * res + rightCol.hashCode(); + res = 31 * res + Boolean.hashCode(left); + + return res; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object obj) { + if (obj instanceof PartitionJoinCondition) { + PartitionJoinCondition other = (PartitionJoinCondition)obj; + + return F.eq(leftAlias, other.leftAlias) && F.eq(rightAlias, other.rightAlias) && + F.eq(leftCol, other.leftCol) && F.eq(rightCol, other.rightCol) && F.eq(left, other.left); + } + + return false; + } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java index ee6bcdecbf63a..e3f47f88c8046 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.ignite.internal.processors.query.h2.affinity.join; import java.util.Collection; @@ -9,7 +26,7 @@ public class PartitionJoinGroup { private final Collection tbls = Collections.newSetFromMap(new IdentityHashMap<>()); /** Tables that were left joined to the group (i.e. these are tables that were on the right side of LJ. */ - private final Collection outerTbls = Collections.newSetFromMap(new IdentityHashMap<>()); + private final Collection outerTbls = Collections.newSetFromMap(new IdentityHashMap<>()); /** Affinity function identifier. */ private final PartitionJoinAffinityIdentifier affIdentifier; @@ -17,8 +34,30 @@ public class PartitionJoinGroup { /** Whether this is replicated group. */ private final boolean replicated; + /** + * Constructor. + * + * @param affIdentifier Affinity identifier. + * @param replicated Replicated flag. + */ public PartitionJoinGroup(PartitionJoinAffinityIdentifier affIdentifier, boolean replicated) { this.affIdentifier = affIdentifier; this.replicated = replicated; } + + public Collection tables() { + return tbls; + } + + public Collection outerTables() { + return outerTbls; + } + + public PartitionJoinAffinityIdentifier affinityIdentifer() { + return affIdentifier; + } + + public boolean replicated() { + return replicated; + } } From 0b016d2054364d0fc254fbe6f9390851339610bc Mon Sep 17 00:00:00 2001 From: devozerov Date: Tue, 11 Dec 2018 18:03:10 +0300 Subject: [PATCH 04/80] WIP. --- .../query/h2/affinity/PartitionExtractor.java | 50 ++----- .../h2/affinity/PartitionExtractorUtils.java | 134 ++++++++++++++++++ .../h2/affinity/join/PartitionJoinGroup.java | 28 ++++ .../h2/affinity/join/PartitionJoinTable.java | 43 ++++-- .../query/h2/sql/GridSqlColumn.java | 7 + .../query/QueryRewritePlaygroundTest.java | 10 +- 6 files changed, 222 insertions(+), 50 deletions(-) create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java index 2ed2d43ae9267..b0139cb57711e 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java @@ -23,7 +23,6 @@ import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinCondition; import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinGroup; import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinModel; -import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor; import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlAlias; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlAst; @@ -39,18 +38,15 @@ import org.apache.ignite.internal.processors.query.h2.sql.GridSqlTable; import org.apache.ignite.internal.util.typedef.F; import org.h2.table.Column; -import org.h2.table.IndexColumn; import org.jetbrains.annotations.Nullable; import java.util.Collection; -import java.util.HashMap; +import java.util.Collections; import java.util.HashSet; +import java.util.IdentityHashMap; import java.util.List; -import java.util.Map; import java.util.Set; -import static org.apache.ignite.internal.processors.query.h2.opt.GridH2KeyValueRowOnheap.DEFAULT_COLUMNS_COUNT; - /** * Partition tree extractor. */ @@ -160,7 +156,7 @@ else if (!F.eq(desc, qryRes.descriptor())) * @return Join model. */ private PartitionJoinModel prepareJoinModel(GridSqlAst from, GridSqlAst where) { - Map grps = new HashMap<>(); + Collection grps = Collections.newSetFromMap(new IdentityHashMap<>()); Collection conds = new HashSet<>(); prepareJoinModelTables(from, grps, conds); @@ -179,7 +175,7 @@ private PartitionJoinModel prepareJoinModel(GridSqlAst from, GridSqlAst where) { */ private boolean prepareJoinModelTables( GridSqlAst from, - Map grps, + Collection grps, Collection conds ) { if (from instanceof GridSqlJoin) { @@ -210,15 +206,15 @@ private boolean prepareJoinModelTables( } } - String alias = null; + PartitionJoinGroup grp = PartitionExtractorUtils.joinGroupForTable(from); - if (from instanceof GridSqlAlias) { - alias = ((GridSqlAlias)from).alias(); + if (grp != null) { + grps.add(grp); - from = from.child(); + return true; } - return true; + return false; } /** @@ -417,7 +413,7 @@ else if (right instanceof GridSqlParameter) { GridH2Table tbl = (GridH2Table)leftCol.getTable(); - if (!isAffinityKey(leftCol.getColumnId(), tbl)) + if (!PartitionExtractorUtils.isAffinityKeyColumn(leftCol, tbl)) return null; PartitionTableDescriptor tblDesc = descriptor(tbl); @@ -433,32 +429,6 @@ else if (rightParam != null) return null; } - /** - * - * @param colId Column ID to check - * @param tbl H2 Table - * @return is affinity key or not - */ - private static boolean isAffinityKey(int colId, GridH2Table tbl) { - GridH2RowDescriptor desc = tbl.rowDescriptor(); - - if (desc.isKeyColumn(colId)) - return true; - - IndexColumn affKeyCol = tbl.getAffinityKeyColumn(); - - try { - return - affKeyCol != null && - colId >= DEFAULT_COLUMNS_COUNT && - desc.isColumnKeyProperty(colId - DEFAULT_COLUMNS_COUNT) && - colId == affKeyCol.column.getColumnId(); - } - catch (IllegalStateException e) { - return false; - } - } - /** * Get descriptor from table. * diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java new file mode 100644 index 0000000000000..aed4ee2e148aa --- /dev/null +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.h2.affinity; + +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinAffinityIdentifier; +import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinGroup; +import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinTable; +import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor; +import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table; +import org.apache.ignite.internal.processors.query.h2.sql.GridSqlAlias; +import org.apache.ignite.internal.processors.query.h2.sql.GridSqlAst; +import org.apache.ignite.internal.processors.query.h2.sql.GridSqlTable; +import org.h2.table.Column; +import org.h2.table.IndexColumn; + +import static org.apache.ignite.internal.processors.query.h2.opt.GridH2KeyValueRowOnheap.DEFAULT_COLUMNS_COUNT; + +/** + * Utility methods for partition extraction. + */ +public class PartitionExtractorUtils { + + public static PartitionJoinGroup joinGroupForTable(GridSqlAst from) { + String alias = null; + + if (from instanceof GridSqlAlias) { + alias = ((GridSqlAlias)from).alias(); + + from = from.child(); + } + + if (from instanceof GridSqlTable) { + GridSqlTable from0 = (GridSqlTable)from; + + GridH2Table tbl0 = from0.dataTable(); + + if (tbl0 == null) + return null; + + // Use identifier string because there might be two table with the same name but form different schemas. + if (alias == null) + alias = tbl0.identifierString(); + + String cacheName = tbl0.cacheName(); + + String affColName = null; + + for (Column col : tbl0.getColumns()) { + // TODO: Wrong! We may have multiple affinity key oclumns here! + if (isAffinityKeyColumn(col, tbl0)) { + affColName = col.getName(); + + break; + } + } + + PartitionJoinTable joinTbl = new PartitionJoinTable(alias, cacheName, affColName); + + CacheConfiguration ccfg = tbl0.cacheInfo().config(); + + PartitionJoinAffinityIdentifier affIdentifier = affinityIdentifierForCache(ccfg); + + // TODO: Wrong. + boolean replicated = affIdentifier != null && ccfg.getCacheMode() == CacheMode.REPLICATED; + + return new PartitionJoinGroup(affIdentifier, replicated).addTable(joinTbl); + } + + return null; + } + + /** + * Prepare affinity identifier for cache. + * + * @param ccfg Cache configuration. + * @return Affinity identifier. + */ + private static PartitionJoinAffinityIdentifier affinityIdentifierForCache(CacheConfiguration ccfg) { + return null; + } + + /** + * Check if the given column is affinity column. + * + * @param col Column. + * @param tbl H2 Table. + * @return is affinity key or not + */ + public static boolean isAffinityKeyColumn(Column col, GridH2Table tbl) { + int colId = col.getColumnId(); + + GridH2RowDescriptor desc = tbl.rowDescriptor(); + + if (desc.isKeyColumn(colId)) + return true; + + IndexColumn affKeyCol = tbl.getAffinityKeyColumn(); + + try { + return + affKeyCol != null && + colId >= DEFAULT_COLUMNS_COUNT && + desc.isColumnKeyProperty(colId - DEFAULT_COLUMNS_COUNT) && + colId == affKeyCol.column.getColumnId(); + } + catch (IllegalStateException e) { + return false; + } + } + + /** + * Private constructor. + */ + private PartitionExtractorUtils() { + // No-op. + } +} diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java index e3f47f88c8046..4bf9f13d3a198 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java @@ -21,6 +21,10 @@ import java.util.Collections; import java.util.IdentityHashMap; +/** + * Partition join group. + */ +@SuppressWarnings("AssignmentOrReturnOfFieldWithMutableType") public class PartitionJoinGroup { /** Tables within a group. */ private final Collection tbls = Collections.newSetFromMap(new IdentityHashMap<>()); @@ -45,18 +49,42 @@ public PartitionJoinGroup(PartitionJoinAffinityIdentifier affIdentifier, boolean this.replicated = replicated; } + /** + * @return Tables in a group. + */ public Collection tables() { return tbls; } + /** + * Add table to the group. + * + * @param tbl Table. + * @return This for chaining. + */ + public PartitionJoinGroup addTable(PartitionJoinTable tbl) { + tbls.add(tbl); + + return this; + } + + /** + * @return Outer tables. + */ public Collection outerTables() { return outerTbls; } + /** + * @return Affinity identifier. + */ public PartitionJoinAffinityIdentifier affinityIdentifer() { return affIdentifier; } + /** + * @return Replicated flag. + */ public boolean replicated() { return replicated; } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java index 2f5b78c0f6477..52c435cc0c4e4 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java @@ -33,23 +33,50 @@ public class PartitionJoinTable { /** Whether table is left joined. */ private boolean leftJoined; - public PartitionJoinTable( - String alias, - String cacheName, - PartitionJoinAffinityIdentifier affIdentifier, - String affColName, - boolean replicated, - boolean subquery - ) { + /** + * Constructor. + * + * @param alias Unique alias. + * @param cacheName Cache name. + * @param affColName Affinity column name. + */ + public PartitionJoinTable(String alias, String cacheName, String affColName) { this.alias = alias; this.cacheName = cacheName; this.affColName = affColName; } + /** + * @return Alias. + */ + public String alias() { + return alias; + } + + /** + * @return Cache name. + */ + public String cacheName() { + return cacheName; + } + + /** + * @return Affinity column name. + */ + public String affinityColName() { + return affColName; + } + + /** + * @param leftJoined Whether table is left-joined. + */ public void leftJoined(boolean leftJoined) { this.leftJoined = leftJoined; } + /** + * @return Whether table is left-joined. + */ public boolean leftJoined() { return leftJoined; } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlColumn.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlColumn.java index a39cf068e0ec6..e66d1efcd1ecb 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlColumn.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlColumn.java @@ -85,6 +85,13 @@ public String schema() { return schema; } + /** + * @return Table alias. + */ + public String tableAlias() { + return tblAlias; + } + /** * @param tblAlias Table alias. */ diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/QueryRewritePlaygroundTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/QueryRewritePlaygroundTest.java index 555f6f9a790cf..55dd601e92451 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/QueryRewritePlaygroundTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/QueryRewritePlaygroundTest.java @@ -42,16 +42,22 @@ public class QueryRewritePlaygroundTest extends GridCommonAbstractTest { */ public void testVarious() throws Exception { executeSql("CREATE TABLE dept (id BIGINT PRIMARY KEY, name VARCHAR)"); - executeSql("CREATE TABLE emp (id BIGINT PRIMARY KEY, name VARCHAR, dept_id BIGINT, salary BIGINT)"); + executeSql("CREATE TABLE emp (id BIGINT, name VARCHAR, dept_id BIGINT, salary BIGINT, PRIMARY KEY (id, dept_id)) WITH \"affinity_key=dept_id\""); //executeSql("SELECT emp.name, dept.name FROM emp, dept WHERE emp.dept_id=dept.id"); - executeSql("SELECT emp.name, dept.name FROM emp INNER JOIN dept ON emp.dept_id=dept.id"); +// executeSql("SELECT emp.name, dept.name FROM emp INNER JOIN dept ON emp.dept_id=dept.id"); // executeSql("" + // "SELECT emp.name, (SELECT dept.name FROM dept WHERE emp.dept_id=dept.id)\n" + // "FROM emp\n" + // "WHERE emp.salary > 1000" // ); + + executeSql("" + + "SELECT * FROM emp e " + + " LEFT JOIN dept d ON e.dept_id = d.id " + + " LEFT JOIN dept d2 ON e.dept_id > d2.id" + ); } public static void executeSql(String sql) throws Exception { From e78ed0c5ce24220874bed476d5613da1057f077f Mon Sep 17 00:00:00 2001 From: devozerov Date: Tue, 18 Dec 2018 11:12:12 +0300 Subject: [PATCH 05/80] Merged with master. --- .../h2/affinity/PartitionExtractorUtils.java | 36 ++----------------- 1 file changed, 2 insertions(+), 34 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java index aed4ee2e148aa..c8c9d34db5d03 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java @@ -22,15 +22,11 @@ import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinAffinityIdentifier; import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinGroup; import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinTable; -import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor; import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlAlias; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlAst; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlTable; import org.h2.table.Column; -import org.h2.table.IndexColumn; - -import static org.apache.ignite.internal.processors.query.h2.opt.GridH2KeyValueRowOnheap.DEFAULT_COLUMNS_COUNT; /** * Utility methods for partition extraction. @@ -64,7 +60,7 @@ public static PartitionJoinGroup joinGroupForTable(GridSqlAst from) { for (Column col : tbl0.getColumns()) { // TODO: Wrong! We may have multiple affinity key oclumns here! - if (isAffinityKeyColumn(col, tbl0)) { + if (tbl0.isColumnForPartitionPruning(col)) { affColName = col.getName(); break; @@ -93,38 +89,10 @@ public static PartitionJoinGroup joinGroupForTable(GridSqlAst from) { * @return Affinity identifier. */ private static PartitionJoinAffinityIdentifier affinityIdentifierForCache(CacheConfiguration ccfg) { + // TODO return null; } - /** - * Check if the given column is affinity column. - * - * @param col Column. - * @param tbl H2 Table. - * @return is affinity key or not - */ - public static boolean isAffinityKeyColumn(Column col, GridH2Table tbl) { - int colId = col.getColumnId(); - - GridH2RowDescriptor desc = tbl.rowDescriptor(); - - if (desc.isKeyColumn(colId)) - return true; - - IndexColumn affKeyCol = tbl.getAffinityKeyColumn(); - - try { - return - affKeyCol != null && - colId >= DEFAULT_COLUMNS_COUNT && - desc.isColumnKeyProperty(colId - DEFAULT_COLUMNS_COUNT) && - colId == affKeyCol.column.getColumnId(); - } - catch (IllegalStateException e) { - return false; - } - } - /** * Private constructor. */ From 456f3440ab3de251ff912533f6c3f1734d392b64 Mon Sep 17 00:00:00 2001 From: devozerov Date: Tue, 18 Dec 2018 12:06:46 +0300 Subject: [PATCH 06/80] Implemented strict and non-strict partition extraction logic. --- .../processors/query/h2/opt/GridH2Table.java | 126 +++++++++++++----- 1 file changed, 92 insertions(+), 34 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java index b69a011e7e8cd..91503772618fa 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java @@ -62,7 +62,6 @@ import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.internal.processors.query.h2.opt.GridH2KeyValueRowOnheap.DEFAULT_COLUMNS_COUNT; -import static org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor.COL_NOT_EXISTS; /** * H2 Table implementation. @@ -101,8 +100,8 @@ public class GridH2Table extends TableBase { /** */ private final IndexColumn affKeyCol; - /** */ - private final int affKeyColId; + /** Whether affinity key column is the whole cache key. */ + private final boolean affKeyColIsKey; /** */ private final LongAdder size = new LongAdder(); @@ -141,36 +140,8 @@ public GridH2Table(CreateTableData createTblData, GridH2RowDescriptor desc, H2Ro this.desc = desc; this.cacheInfo = cacheInfo; - if (!desc.type().customAffinityKeyMapper()) { - String affKeyFieldName = desc.type().affinityKey(); - - if (affKeyFieldName != null) { - if (doesColumnExist(affKeyFieldName)) { - int colId = getColumn(affKeyFieldName).getColumnId(); - - if (desc.isKeyColumn(colId)) { - affKeyCol = indexColumn(GridH2KeyValueRowOnheap.KEY_COL, SortOrder.ASCENDING); - affKeyColId = GridH2KeyValueRowOnheap.KEY_COL; - } - else { - affKeyCol = indexColumn(colId, SortOrder.ASCENDING); - affKeyColId = colId; - } - } - else { - affKeyCol = null; - affKeyColId = COL_NOT_EXISTS; - } - } - else { - affKeyCol = indexColumn(GridH2KeyValueRowOnheap.KEY_COL, SortOrder.ASCENDING); - affKeyColId = GridH2KeyValueRowOnheap.KEY_COL; - } - } - else { - affKeyCol = null; - affKeyColId = COL_NOT_EXISTS; - } + affKeyCol = calculateAffinityKeyColumn(); + affKeyColIsKey = affKeyCol != null && desc.isKeyColumn(affKeyCol.column.getColumnId()); this.rowFactory = rowFactory; @@ -209,6 +180,36 @@ public GridH2Table(CreateTableData createTblData, GridH2RowDescriptor desc, H2Ro lock = new ReentrantReadWriteLock(); } + /** + * Calculate affinity key column which will be used for partition pruning and distributed joins. + * + * @return Affinity column or {@code null} if none can be used. + */ + private IndexColumn calculateAffinityKeyColumn() { + // If custome affinity key mapper is set, we do not know how to convert _KEY to partition, return null. + if (desc.type().customAffinityKeyMapper()) + return null; + + String affKeyFieldName = desc.type().affinityKey(); + + // If explicit affinity key field is not set, then use _KEY. + if (affKeyFieldName == null) + return indexColumn(GridH2KeyValueRowOnheap.KEY_COL, SortOrder.ASCENDING); + + // If explicit affinity key field is set, but is not found in the table, do not use anything. + if (!doesColumnExist(affKeyFieldName)) + return null; + + int colId = getColumn(affKeyFieldName).getColumnId(); + + // If affinity key column is either _KEY or it's alias (QueryEntity.keyFieldName), normalize it to _KEY. + if (desc.isKeyColumn(colId)) + return indexColumn(GridH2KeyValueRowOnheap.KEY_COL, SortOrder.ASCENDING); + + // Otherwise use column as is. + return indexColumn(colId, SortOrder.ASCENDING); + } + /** * @return {@code true} If this is a partitioned table. */ @@ -230,9 +231,65 @@ public boolean isPartitioned() { * @return {@code True} if affinity key column. */ public boolean isColumnForPartitionPruning(Column col) { + return isColumnForPartitionPruning0(col, false); + } + + /** + * Check whether passed column could be used for partition transfer during partition pruning on joined tables and + * for external affinity calculation (e.g. on thin clients). + *

+ * Note that it is different from {@link #isColumnForPartitionPruning(Column)} method in that not every column + * which qualifies for partition pruning can be used by thin clients or join partinion prunining logic. + *

+ * Consider the following schema: + * 

+     * CREATE TABLE dept (id PRIMARY KEY);
+     * CREATE TABLE emp (id, dept_id AFFINITY KEY, PRIMARY KEY(id, dept_id));
+     *
+ * For expression-based partition pruning on "emp" table on the server side we may use both "_KEY" and + * "dept_id" columns, as passing them through standard affinity workflow will yield the same result: + * dept_id -> part + * _KEY -> dept_id -> part + *

+ * But we cannot use "_KEY" on thin client side, as it doesn't know how to extract affinity key field properly. + * Neither we can perform partition transfer in JOINs when "_KEY" is used. + *

+ * This is OK as data is collocated, so we can merge partitions extracted from both tables: + * 

+     * SELECT * FROM dept d INNER JOIN emp e ON d.id = e.dept_id WHERE e.dept_id=? AND d.id=?
+     *
+ * But this is not OK as joined data is not collocated, and tables form distinct collocation groups: + * 
+     * SELECT * FROM dept d INNER JOIN emp e ON d.id = e._KEY WHERE e.dept_id=? AND d.id=?
+     *
+ * NB: The last query is not logically correct and will produce empty result. However, it is correct from SQL + * perspective, so we should make incorrect assumptions about partitions as ti may make situation even worse. + * + * @param col Column. + * @return {@code True} if column could be used for partition extraction on both server and client sides and for + * partition transfer in joins. + */ + public boolean isColumnForPartitionPruningStrict(Column col) { + return isColumnForPartitionPruning0(col, true); + } + + /** + * Internal logic to check whether column qualifies for partition extraction or not. + * + * @param col Column. + * @param strict Strict flag. + * @return {@code True} if column could be used for partition. + */ + private boolean isColumnForPartitionPruning0(Column col, boolean strict) { + if (affKeyCol == null) + return false; + int colId = col.getColumnId(); - return colId == affKeyColId || desc.isKeyColumn(colId); + if (colId == affKeyCol.column.getColumnId()) + return true; + + return (affKeyColIsKey || !strict) && desc.isKeyColumn(colId); } /** @@ -333,6 +390,7 @@ public String identifierString() { * * @param exclusive Exclusive flag. */ + @SuppressWarnings({"LockAcquiredButNotSafelyReleased", "CallToThreadYield"}) private void lock(boolean exclusive) { Lock l = exclusive ? lock.writeLock() : lock.readLock(); From ca318d4e6f28fa121705fc8289e263a3a5ac2472 Mon Sep 17 00:00:00 2001 From: devozerov Date: Tue, 18 Dec 2018 12:16:33 +0300 Subject: [PATCH 07/80] Affinity column name resolution for model. --- .../h2/affinity/PartitionExtractorUtils.java | 17 +++++++---- .../h2/affinity/join/PartitionJoinTable.java | 30 +++++++++++++++++-- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java index c8c9d34db5d03..5b5aadb2b0b86 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java @@ -57,17 +57,22 @@ public static PartitionJoinGroup joinGroupForTable(GridSqlAst from) { String cacheName = tbl0.cacheName(); String affColName = null; + String secondAffColName = null; for (Column col : tbl0.getColumns()) { - // TODO: Wrong! We may have multiple affinity key oclumns here! - if (tbl0.isColumnForPartitionPruning(col)) { - affColName = col.getName(); - - break; + if (tbl0.isColumnForPartitionPruningStrict(col)) { + if (affColName == null) + affColName = col.getName(); + else { + secondAffColName = col.getName(); + + // Break as we cannot have more than two affinity key columns. + break; + } } } - PartitionJoinTable joinTbl = new PartitionJoinTable(alias, cacheName, affColName); + PartitionJoinTable joinTbl = new PartitionJoinTable(alias, cacheName, affColName, secondAffColName); CacheConfiguration ccfg = tbl0.cacheInfo().config(); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java index 52c435cc0c4e4..518d7226fd773 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java @@ -17,6 +17,8 @@ package org.apache.ignite.internal.processors.query.h2.affinity.join; +import org.jetbrains.annotations.Nullable; + /** * Single table with affinity info. */ @@ -30,6 +32,9 @@ public class PartitionJoinTable { /** Affinity column name (if can be resolved). */ private final String affColName; + /** Second affinity column name (possible when _KEY is affinity column and an alias for this column exists. */ + private final String secondAffColName; + /** Whether table is left joined. */ private boolean leftJoined; @@ -39,11 +44,21 @@ public class PartitionJoinTable { * @param alias Unique alias. * @param cacheName Cache name. * @param affColName Affinity column name. + * @param secondAffColName Second affinity column name. */ - public PartitionJoinTable(String alias, String cacheName, String affColName) { + public PartitionJoinTable(String alias, String cacheName, @Nullable String affColName, + @Nullable String secondAffColName) { this.alias = alias; this.cacheName = cacheName; - this.affColName = affColName; + + if (affColName == null && secondAffColName != null) { + this.affColName = secondAffColName; + this.secondAffColName = null; + } + else { + this.affColName = affColName; + this.secondAffColName = secondAffColName; + } } /** @@ -60,6 +75,10 @@ public String cacheName() { return cacheName; } + public boolean hasAffinityColumn() { + return affColName != null; + } + /** * @return Affinity column name. */ @@ -67,6 +86,13 @@ public String affinityColName() { return affColName; } + /** + * @return Second affinity column name. + */ + public String secondAffinityColName() { + return secondAffColName; + } + /** * @param leftJoined Whether table is left-joined. */ From 2fa20fb1c876ecfaf6059ffd292a912db5c69b71 Mon Sep 17 00:00:00 2001 From: devozerov Date: Tue, 18 Dec 2018 13:30:17 +0300 Subject: [PATCH 08/80] Affinity descriptor. --- .../h2/affinity/PartitionExtractorUtils.java | 6 +- ...ava => PartitionAffinityFunctionType.java} | 43 ++++++++- .../join/PartitionJoinAffinityDescriptor.java | 96 +++++++++++++++++++ .../h2/affinity/join/PartitionJoinGroup.java | 6 +- 4 files changed, 142 insertions(+), 9 deletions(-) rename modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/{PartitionJoinAffinityIdentifier.java => PartitionAffinityFunctionType.java} (50%) create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinAffinityDescriptor.java diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java index 5b5aadb2b0b86..ab58bbc2c5f36 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java @@ -19,7 +19,7 @@ import org.apache.ignite.cache.CacheMode; import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinAffinityIdentifier; +import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinAffinityDescriptor; import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinGroup; import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinTable; import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table; @@ -76,7 +76,7 @@ public static PartitionJoinGroup joinGroupForTable(GridSqlAst from) { CacheConfiguration ccfg = tbl0.cacheInfo().config(); - PartitionJoinAffinityIdentifier affIdentifier = affinityIdentifierForCache(ccfg); + PartitionJoinAffinityDescriptor affIdentifier = affinityIdentifierForCache(ccfg); // TODO: Wrong. boolean replicated = affIdentifier != null && ccfg.getCacheMode() == CacheMode.REPLICATED; @@ -93,7 +93,7 @@ public static PartitionJoinGroup joinGroupForTable(GridSqlAst from) { * @param ccfg Cache configuration. * @return Affinity identifier. */ - private static PartitionJoinAffinityIdentifier affinityIdentifierForCache(CacheConfiguration ccfg) { + private static PartitionJoinAffinityDescriptor affinityIdentifierForCache(CacheConfiguration ccfg) { // TODO return null; } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinAffinityIdentifier.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionAffinityFunctionType.java similarity index 50% rename from modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinAffinityIdentifier.java rename to modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionAffinityFunctionType.java index 1dc2439c90177..58f62abf6e83a 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinAffinityIdentifier.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionAffinityFunctionType.java @@ -17,12 +17,49 @@ package org.apache.ignite.internal.processors.query.h2.affinity.join; -import java.io.Serializable; +import org.jetbrains.annotations.Nullable; /** - * Affinity function identifier. Used to compare affinity functions of two tables. + * Affinity function type. */ -public interface PartitionJoinAffinityIdentifier extends Serializable { +public enum PartitionAffinityFunctionType { + /** Custom affintiy function. */ + CUSTOM(0), + /** Rendezvous affinity function. */ + RENDEZVOUS(1); + /** Value. */ + private final int val; + + /** + * Constructor. + * + * @param val Value. + */ + PartitionAffinityFunctionType(int val) { + this.val = val; + } + + /** + * @return Value. + */ + public int value() { + return val; + } + + /** + * Get type from value. + * + * @param val Value. + * @return Type or [@code null} if cannot be resolved.. + */ + @Nullable public static PartitionAffinityFunctionType fromValue(int val) { + for (PartitionAffinityFunctionType typ : values()) { + if (typ.val == val) + return typ; + } + + return null; + } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinAffinityDescriptor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinAffinityDescriptor.java new file mode 100644 index 0000000000000..108535c0bf02e --- /dev/null +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinAffinityDescriptor.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.h2.affinity.join; + +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.internal.util.typedef.internal.S; + +import java.io.Serializable; + +/** + * Affinity function descriptor. Used to compare affinity functions of two tables. + */ +public class PartitionJoinAffinityDescriptor implements Serializable { + /** */ + private static final long serialVersionUID = 0L; + + /** Cache mode. */ + private final CacheMode cacheMode; + + /** Affinity function type. */ + private final PartitionAffinityFunctionType affFunc; + + /** Number of partitions. */ + private final int parts; + + /** Whether node filter is set. */ + private final boolean hasNodeFilter; + + /** + * Constructor. + * + * @param cacheMode Cache mode. + * @param affFunc Affinity function type. + * @param parts Number of partitions. + * @param hasNodeFilter Whether node filter is set. + */ + public PartitionJoinAffinityDescriptor( + CacheMode cacheMode, + PartitionAffinityFunctionType affFunc, + int parts, + boolean hasNodeFilter + ) { + this.cacheMode = cacheMode; + this.affFunc = affFunc; + this.parts = parts; + this.hasNodeFilter = hasNodeFilter; + } + + /** + * Check is provided descriptor is compatible with this instance (i.e. can be used in the same co-location group). + * + * @param other Other descriptor. + * @return {@code True} if compatible. + */ + public boolean isCompatible(PartitionJoinAffinityDescriptor other) { + // REPLICATED caches has special treatment during parititon pruning, so exclude them. + if (cacheMode == CacheMode.PARTITIONED) { + // Rendezvous affinity function is deterministic and doesn't depend on previous cluster view changes. + // In future other user affinity functions would be applicable as well if explicityl marked deterministic. + if (affFunc == PartitionAffinityFunctionType.RENDEZVOUS) { + // We cannot be sure that two caches are co-located if custom node filter is present. + // Nota that technically we may try to compare two filters. However, this adds unnecessary complexity + // and potential deserialization issues when SQL is called from client nodes or thin clients. + if (!hasNodeFilter) { + return + other.cacheMode == CacheMode.PARTITIONED && + other.affFunc == PartitionAffinityFunctionType.RENDEZVOUS && + !other.hasNodeFilter && + other.parts == parts; + } + } + } + + return false; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(PartitionJoinAffinityDescriptor.class, this); + } +} diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java index 4bf9f13d3a198..bb808bbee6188 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java @@ -33,7 +33,7 @@ public class PartitionJoinGroup { private final Collection outerTbls = Collections.newSetFromMap(new IdentityHashMap<>()); /** Affinity function identifier. */ - private final PartitionJoinAffinityIdentifier affIdentifier; + private final PartitionJoinAffinityDescriptor affIdentifier; /** Whether this is replicated group. */ private final boolean replicated; @@ -44,7 +44,7 @@ public class PartitionJoinGroup { * @param affIdentifier Affinity identifier. * @param replicated Replicated flag. */ - public PartitionJoinGroup(PartitionJoinAffinityIdentifier affIdentifier, boolean replicated) { + public PartitionJoinGroup(PartitionJoinAffinityDescriptor affIdentifier, boolean replicated) { this.affIdentifier = affIdentifier; this.replicated = replicated; } @@ -78,7 +78,7 @@ public Collection outerTables() { /** * @return Affinity identifier. */ - public PartitionJoinAffinityIdentifier affinityIdentifer() { + public PartitionJoinAffinityDescriptor affinityIdentifer() { return affIdentifier; } From e1ccb5af84f938f9f609724a7c7131f0a2f24e77 Mon Sep 17 00:00:00 2001 From: devozerov Date: Tue, 18 Dec 2018 15:02:29 +0300 Subject: [PATCH 09/80] Join model for single table. --- .../h2/affinity/PartitionExtractorUtils.java | 41 +++++++++++++------ .../h2/affinity/join/PartitionJoinGroup.java | 28 ++++--------- .../h2/affinity/join/PartitionJoinTable.java | 38 ++++++++++++++++- 3 files changed, 72 insertions(+), 35 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java index ab58bbc2c5f36..fb5617253ae01 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java @@ -17,8 +17,9 @@ package org.apache.ignite.internal.processors.query.h2.affinity; -import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionAffinityFunctionType; import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinAffinityDescriptor; import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinGroup; import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinTable; @@ -33,6 +34,12 @@ */ public class PartitionExtractorUtils { + /** + * Prepare join group for a single table. + * + * @param from Table. + * @return Join group. + */ public static PartitionJoinGroup joinGroupForTable(GridSqlAst from) { String alias = null; @@ -43,12 +50,14 @@ public static PartitionJoinGroup joinGroupForTable(GridSqlAst from) { } if (from instanceof GridSqlTable) { + // Normal table. GridSqlTable from0 = (GridSqlTable)from; GridH2Table tbl0 = from0.dataTable(); if (tbl0 == null) - return null; + // Unknown table type, e.g. temp table. + return new PartitionJoinGroup(null).addTable(new PartitionJoinTable(alias)); // Use identifier string because there might be two table with the same name but form different schemas. if (alias == null) @@ -74,17 +83,16 @@ public static PartitionJoinGroup joinGroupForTable(GridSqlAst from) { PartitionJoinTable joinTbl = new PartitionJoinTable(alias, cacheName, affColName, secondAffColName); - CacheConfiguration ccfg = tbl0.cacheInfo().config(); + PartitionJoinAffinityDescriptor affDesc = affinityDescriptorForCache(tbl0.cacheInfo().config()); - PartitionJoinAffinityDescriptor affIdentifier = affinityIdentifierForCache(ccfg); - - // TODO: Wrong. - boolean replicated = affIdentifier != null && ccfg.getCacheMode() == CacheMode.REPLICATED; - - return new PartitionJoinGroup(affIdentifier, replicated).addTable(joinTbl); + return new PartitionJoinGroup(affDesc).addTable(joinTbl); } + else { + // Subquery/union + assert alias != null; - return null; + return new PartitionJoinGroup(null).addTable(new PartitionJoinTable(alias)); + } } /** @@ -93,9 +101,16 @@ public static PartitionJoinGroup joinGroupForTable(GridSqlAst from) { * @param ccfg Cache configuration. * @return Affinity identifier. */ - private static PartitionJoinAffinityDescriptor affinityIdentifierForCache(CacheConfiguration ccfg) { - // TODO - return null; + private static PartitionJoinAffinityDescriptor affinityDescriptorForCache(CacheConfiguration ccfg) { + PartitionAffinityFunctionType aff = ccfg.getAffinity().getClass().equals(RendezvousAffinityFunction.class) ? + PartitionAffinityFunctionType.RENDEZVOUS : PartitionAffinityFunctionType.CUSTOM; + + return new PartitionJoinAffinityDescriptor( + ccfg.getCacheMode(), + aff, + ccfg.getAffinity().partitions(), + ccfg.getNodeFilter() != null + ); } /** diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java index bb808bbee6188..72fe444b2dfac 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java @@ -32,21 +32,16 @@ public class PartitionJoinGroup { /** Tables that were left joined to the group (i.e. these are tables that were on the right side of LJ. */ private final Collection outerTbls = Collections.newSetFromMap(new IdentityHashMap<>()); - /** Affinity function identifier. */ - private final PartitionJoinAffinityDescriptor affIdentifier; - - /** Whether this is replicated group. */ - private final boolean replicated; + /** Affinity function descriptor. */ + private final PartitionJoinAffinityDescriptor affDesc; /** * Constructor. * - * @param affIdentifier Affinity identifier. - * @param replicated Replicated flag. + * @param affDesc Affinity function descriptor. */ - public PartitionJoinGroup(PartitionJoinAffinityDescriptor affIdentifier, boolean replicated) { - this.affIdentifier = affIdentifier; - this.replicated = replicated; + public PartitionJoinGroup(PartitionJoinAffinityDescriptor affDesc) { + this.affDesc = affDesc; } /** @@ -76,16 +71,9 @@ public Collection outerTables() { } /** - * @return Affinity identifier. - */ - public PartitionJoinAffinityDescriptor affinityIdentifer() { - return affIdentifier; - } - - /** - * @return Replicated flag. + * @return Affinity descriptor. */ - public boolean replicated() { - return replicated; + public PartitionJoinAffinityDescriptor affinityDescriptor() { + return affDesc; } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java index 518d7226fd773..753f2419b934e 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java @@ -35,9 +35,27 @@ public class PartitionJoinTable { /** Second affinity column name (possible when _KEY is affinity column and an alias for this column exists. */ private final String secondAffColName; + /** Whether this is not a classical table. */ + private final boolean nonTable; + /** Whether table is left joined. */ private boolean leftJoined; + /** + * Create join table for subquery. + * + * @param alias Alias. + */ + public PartitionJoinTable(String alias) { + this.alias = alias; + + cacheName = null; + affColName = null; + secondAffColName = null; + + nonTable = true; + } + /** * Constructor. * @@ -46,8 +64,12 @@ public class PartitionJoinTable { * @param affColName Affinity column name. * @param secondAffColName Second affinity column name. */ - public PartitionJoinTable(String alias, String cacheName, @Nullable String affColName, - @Nullable String secondAffColName) { + public PartitionJoinTable( + String alias, + String cacheName, + @Nullable String affColName, + @Nullable String secondAffColName + ) { this.alias = alias; this.cacheName = cacheName; @@ -59,6 +81,8 @@ public PartitionJoinTable(String alias, String cacheName, @Nullable String affCo this.affColName = affColName; this.secondAffColName = secondAffColName; } + + nonTable = false; } /** @@ -75,6 +99,9 @@ public String cacheName() { return cacheName; } + /** + * @return {@code True} if affinity oclumn exists. + */ public boolean hasAffinityColumn() { return affColName != null; } @@ -106,4 +133,11 @@ public void leftJoined(boolean leftJoined) { public boolean leftJoined() { return leftJoined; } + + /** + * @return Whether this is not a classical table (subquery, union, temp tables, etc). + */ + public boolean isNonTable() { + return nonTable; + } } From 0945c74fe71bea5cefb8678fce1ee2b59c575250 Mon Sep 17 00:00:00 2001 From: devozerov Date: Tue, 18 Dec 2018 15:39:40 +0300 Subject: [PATCH 10/80] WIP. --- .../query/h2/affinity/PartitionExtractor.java | 31 ++++++++---- .../h2/affinity/PartitionExtractorUtils.java | 49 +++++++++++++++++++ 2 files changed, 71 insertions(+), 9 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java index ee45c37c94b67..4aaf697fbd351 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java @@ -174,7 +174,7 @@ private PartitionJoinModel prepareJoinModel(GridSqlAst from, GridSqlAst where) { * @param conds Conditions. * @return {@code True} if extracted tables successfully, {@code false} if failed to extract. */ - private boolean prepareJoinModelTables( + private static boolean prepareJoinModelTables( GridSqlAst from, Collection grps, Collection conds @@ -189,9 +189,16 @@ private boolean prepareJoinModelTables( if (!prepareJoinModelTables(join.rightTable(), grps, conds)) return false; - // Make sure that ON condition is simple equality. Stop process otherwise. + // Extract condition from JOIN. Only equijoins are supported for now. + // Note that most conditions are moved to WHERE clause through AND predicate, + // so if it is present, then most likely we deal with LEFT JOIN. GridSqlElement on = join.on(); + if (PartitionExtractorUtils.isCrossJoinCondition(on)) { + PartitionJoinCondition cond = new PartitionJoinCondition(join.leftTable()) + } + + boolean onSimple = false; if (on instanceof GridSqlOperation) { @@ -209,13 +216,9 @@ private boolean prepareJoinModelTables( PartitionJoinGroup grp = PartitionExtractorUtils.joinGroupForTable(from); - if (grp != null) { - grps.add(grp); + grps.add(grp); - return true; - } - - return false; + return true; } /** @@ -441,13 +444,23 @@ private static PartitionTableDescriptor descriptor(GridH2Table tbl) { return new PartitionTableDescriptor(tbl.cacheName(), tbl.getName()); } + /** + * Unwrap constant if possible. + * + * @param ast AST. + * @return Constant or {@code null} if not a constant. + */ + @Nullable public static GridSqlConst unwrapConst(GridSqlAst ast) { + return ast instanceof GridSqlConst ? (GridSqlConst)ast : null; + } + /** * Unwrap column if possible. * * @param ast AST. * @return Column or {@code null} if not a column. */ - @Nullable private static GridSqlColumn unwrapColumn(GridSqlAst ast) { + @Nullable public static GridSqlColumn unwrapColumn(GridSqlAst ast) { if (ast instanceof GridSqlAlias) ast = ast.child(); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java index fb5617253ae01..cb6309e5f8a29 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java @@ -21,13 +21,20 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionAffinityFunctionType; import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinAffinityDescriptor; +import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinCondition; import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinGroup; import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinTable; import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlAlias; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlAst; +import org.apache.ignite.internal.processors.query.h2.sql.GridSqlColumn; +import org.apache.ignite.internal.processors.query.h2.sql.GridSqlConst; +import org.apache.ignite.internal.processors.query.h2.sql.GridSqlElement; +import org.apache.ignite.internal.processors.query.h2.sql.GridSqlOperation; +import org.apache.ignite.internal.processors.query.h2.sql.GridSqlOperationType; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlTable; import org.h2.table.Column; +import org.h2.value.Value; /** * Utility methods for partition extraction. @@ -95,6 +102,48 @@ public static PartitionJoinGroup joinGroupForTable(GridSqlAst from) { } } + /** + * Check whether this is a cross-join condition, i.e. 1=1. + * + * @param on Condition. + * @return {@code True} if cross-join. + */ + public static boolean isCrossJoinCondition(GridSqlAst on) { + if (on instanceof GridSqlOperation) { + GridSqlOperation on0 = (GridSqlOperation)on; + + if (on0.operationType() == GridSqlOperationType.EQUAL) { + GridSqlConst left = PartitionExtractor.unwrapConst(on0.child(0)); + GridSqlConst right = PartitionExtractor.unwrapConst(on0.child(1)); + + if (left != null && right != null) { + try { + int leftVal = left.value().getInt(); + int rightVal = right.value().getInt(); + + return leftVal == rightVal; + } + catch (Exception ignore) { + // No-op. + } + } + } + } + + return false; + } + + /** + * Try parsing condition as simple JOIN codition. Only equijoins are supported for now, so anything more complex + * than "A.a = B.b" are not processed. + * + * @param cond Initial AST. + * @return Join condition or {@code null} if not simple equijoin. + */ + public static PartitionJoinCondition tryParseJoinCondition(GridSqlElement cond) { + + } + /** * Prepare affinity identifier for cache. * From 6b630f25bda2a1e1c6317a65a73d3666d7d15476 Mon Sep 17 00:00:00 2001 From: devozerov Date: Tue, 18 Dec 2018 17:55:26 +0300 Subject: [PATCH 11/80] WIP. --- .../query/h2/affinity/PartitionExtractor.java | 88 +++++++++++++------ .../h2/affinity/PartitionExtractorUtils.java | 23 ++++- .../affinity/join/PartitionJoinCondition.java | 17 +--- .../h2/affinity/join/PartitionJoinGroup.java | 10 --- .../h2/affinity/join/PartitionJoinTable.java | 17 ---- 5 files changed, 82 insertions(+), 73 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java index 4aaf697fbd351..6850d5a6cb386 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java @@ -23,6 +23,7 @@ import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinCondition; import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinGroup; import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinModel; +import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinTable; import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlAlias; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlAst; @@ -40,10 +41,10 @@ import org.h2.table.Column; import org.jetbrains.annotations.Nullable; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashSet; -import java.util.IdentityHashMap; import java.util.List; import java.util.Set; @@ -157,68 +158,97 @@ else if (!F.eq(desc, qryRes.descriptor())) * @return Join model. */ private PartitionJoinModel prepareJoinModel(GridSqlAst from, GridSqlAst where) { - Collection grps = Collections.newSetFromMap(new IdentityHashMap<>()); + Set leftJoined = new HashSet<>(); Collection conds = new HashSet<>(); - prepareJoinModelTables(from, grps, conds); + Collection grps = prepareJoinModelTables(from, leftJoined, conds); // TODO return null; } + /** + * Prepare expressions from WHERE condition. + *

+ * All conditions which qualifies for joined tables must be located at the top level of conjunctive expression. + * That is, (joinCond AND ...) is OK for us. But (joinCond OR ...) isn't. + *

+ * Also we do not expect any additional join expressions on LEFT JOINed columns, so if they are met, processing is + * stopped. + * + * @param where WHERE condition. + * @param leftJoined Left joined tables. + * @param conds Condirtions. + */ + private boolean prepareJoinModelConditions(GridSqlAst where, Collection leftJoined, + Collection conds) { + // TODO + return false; + } + /** * Prepare tables which will be used in join model. * * @param from From flag. - * @param grps Groups. * @param conds Conditions. * @return {@code True} if extracted tables successfully, {@code false} if failed to extract. */ - private static boolean prepareJoinModelTables( + private List prepareJoinModelTables( GridSqlAst from, - Collection grps, + Collection leftJoined, Collection conds ) { if (from instanceof GridSqlJoin) { // Process JOIN recursively. GridSqlJoin join = (GridSqlJoin)from; - if (!prepareJoinModelTables(join.leftTable(), grps, conds)) - return false; + List leftGrp = prepareJoinModelTables(join.leftTable(), leftJoined, conds); - if (!prepareJoinModelTables(join.rightTable(), grps, conds)) - return false; + if (leftGrp == null) + return null; - // Extract condition from JOIN. Only equijoins are supported for now. - // Note that most conditions are moved to WHERE clause through AND predicate, - // so if it is present, then most likely we deal with LEFT JOIN. - GridSqlElement on = join.on(); + List rightGrp = prepareJoinModelTables(join.leftTable(), leftJoined, conds); - if (PartitionExtractorUtils.isCrossJoinCondition(on)) { - PartitionJoinCondition cond = new PartitionJoinCondition(join.leftTable()) - } + if (rightGrp == null) + return null; + + if (join.isLeftOuter()) { + // Do not support complex LEFT-RIGHT join variations for now. + if (rightGrp.size() != 1) + return null; + // "a LEFT JOIN b" is transformed into "a", and "b" is put into special stop-list. + // If a condition is met on "b" afterwards, we will stop partition pruning process. + for (PartitionJoinTable rightTbl : rightGrp.get(0).tables()) + leftJoined.add(rightTbl.alias()); - boolean onSimple = false; + return leftGrp; + } - if (on instanceof GridSqlOperation) { - GridSqlOperation on0 = (GridSqlOperation)on; + // Extract condition from JOIN. Only equijoins are supported for now. + GridSqlElement on = join.on(); - if (on0.operationType() == GridSqlOperationType.EQUAL) { - GridSqlColumn left = unwrapColumn(on0.child(0)); - GridSqlColumn right = unwrapColumn(on0.child(1)); + if (!PartitionExtractorUtils.isCrossJoinCondition(on)) { + PartitionJoinCondition cond = PartitionExtractorUtils.tryParseEquiJoinCondition(on); - // TODO - System.out.println(left + " " + right); - } + if (cond != null) + conds.add(cond); + else + // Non equi-join, stop partition pruning. + return null; } + + ArrayList res = new ArrayList<>(leftGrp.size() + rightGrp.size()); + + res.addAll(leftGrp); + res.addAll(rightGrp); + + return res; } PartitionJoinGroup grp = PartitionExtractorUtils.joinGroupForTable(from); - grps.add(grp); - - return true; + return Collections.singletonList(grp); } /** diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java index cb6309e5f8a29..1aa8605ab403f 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java @@ -137,11 +137,30 @@ public static boolean isCrossJoinCondition(GridSqlAst on) { * Try parsing condition as simple JOIN codition. Only equijoins are supported for now, so anything more complex * than "A.a = B.b" are not processed. * - * @param cond Initial AST. + * @param on Initial AST. * @return Join condition or {@code null} if not simple equijoin. */ - public static PartitionJoinCondition tryParseJoinCondition(GridSqlElement cond) { + public static PartitionJoinCondition tryParseEquiJoinCondition(GridSqlElement on) { + if (on instanceof GridSqlOperation) { + GridSqlOperation on0 = (GridSqlOperation)on; + + if (on0.operationType() == GridSqlOperationType.EQUAL) { + GridSqlColumn left = PartitionExtractor.unwrapColumn(on0.child(0)); + GridSqlColumn right = PartitionExtractor.unwrapColumn(on0.child(1)); + + if (left != null && right != null) { + String leftAlias = left.tableAlias(); + String rightAlias = right.tableAlias(); + + String leftCol = left.columnName(); + String rightCol = right.columnName(); + + return new PartitionJoinCondition(leftAlias, rightAlias, leftCol, rightCol); + } + } + } + return null; } /** diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinCondition.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinCondition.java index b2d3f0d347692..0975452883685 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinCondition.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinCondition.java @@ -35,9 +35,6 @@ public class PartitionJoinCondition { /** Right column name. */ private final String rightCol; - /** Whether this is LEFT OUTER JOIN. */ - private final boolean left; - /** * Constructor. * @@ -45,14 +42,12 @@ public class PartitionJoinCondition { * @param rightAlias Right alias. * @param leftCol Left column name. * @param rightCol Right column name. - * @param left Left join flag. */ - public PartitionJoinCondition(String leftAlias, String rightAlias, String leftCol, String rightCol, boolean left) { + public PartitionJoinCondition(String leftAlias, String rightAlias, String leftCol, String rightCol) { this.leftAlias = leftAlias; this.rightAlias = rightAlias; this.leftCol = leftCol; this.rightCol = rightCol; - this.left = left; } /** @@ -83,13 +78,6 @@ public String rightColumn() { return rightCol; } - /** - * @return Whether this is left join. - */ - public boolean left() { - return left; - } - /** {@inheritDoc} */ @Override public int hashCode() { int res = leftAlias.hashCode(); @@ -97,7 +85,6 @@ public boolean left() { res = 31 * res + rightAlias.hashCode(); res = 31 * res + leftCol.hashCode(); res = 31 * res + rightCol.hashCode(); - res = 31 * res + Boolean.hashCode(left); return res; } @@ -108,7 +95,7 @@ public boolean left() { PartitionJoinCondition other = (PartitionJoinCondition)obj; return F.eq(leftAlias, other.leftAlias) && F.eq(rightAlias, other.rightAlias) && - F.eq(leftCol, other.leftCol) && F.eq(rightCol, other.rightCol) && F.eq(left, other.left); + F.eq(leftCol, other.leftCol) && F.eq(rightCol, other.rightCol); } return false; diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java index 72fe444b2dfac..09d6d8e50ac23 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java @@ -29,9 +29,6 @@ public class PartitionJoinGroup { /** Tables within a group. */ private final Collection tbls = Collections.newSetFromMap(new IdentityHashMap<>()); - /** Tables that were left joined to the group (i.e. these are tables that were on the right side of LJ. */ - private final Collection outerTbls = Collections.newSetFromMap(new IdentityHashMap<>()); - /** Affinity function descriptor. */ private final PartitionJoinAffinityDescriptor affDesc; @@ -63,13 +60,6 @@ public PartitionJoinGroup addTable(PartitionJoinTable tbl) { return this; } - /** - * @return Outer tables. - */ - public Collection outerTables() { - return outerTbls; - } - /** * @return Affinity descriptor. */ diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java index 753f2419b934e..f60ffab41f15a 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java @@ -38,9 +38,6 @@ public class PartitionJoinTable { /** Whether this is not a classical table. */ private final boolean nonTable; - /** Whether table is left joined. */ - private boolean leftJoined; - /** * Create join table for subquery. * @@ -120,20 +117,6 @@ public String secondAffinityColName() { return secondAffColName; } - /** - * @param leftJoined Whether table is left-joined. - */ - public void leftJoined(boolean leftJoined) { - this.leftJoined = leftJoined; - } - - /** - * @return Whether table is left-joined. - */ - public boolean leftJoined() { - return leftJoined; - } - /** * @return Whether this is not a classical table (subquery, union, temp tables, etc). */ From babf609f0c2f0205fca83a0abe79dc2385b30782 Mon Sep 17 00:00:00 2001 From: devozerov Date: Thu, 20 Dec 2018 08:26:07 +0300 Subject: [PATCH 12/80] Table model class. --- .../query/h2/affinity/PartitionExtractor.java | 4 +- .../h2/affinity/join/PartitionJoinModel.java | 24 ---- .../h2/affinity/join/PartitionJoinTable.java | 49 ++++---- .../h2/affinity/join/PartitionTableModel.java | 107 ++++++++++++++++++ 4 files changed, 134 insertions(+), 50 deletions(-) delete mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinModel.java create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java index 6850d5a6cb386..07ac1114c9844 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java @@ -22,7 +22,7 @@ import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing; import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinCondition; import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinGroup; -import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinModel; +import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionTableModel; import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinTable; import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlAlias; @@ -157,7 +157,7 @@ else if (!F.eq(desc, qryRes.descriptor())) * @param where WHERE clause. * @return Join model. */ - private PartitionJoinModel prepareJoinModel(GridSqlAst from, GridSqlAst where) { + private PartitionTableModel prepareJoinModel(GridSqlAst from, GridSqlAst where) { Set leftJoined = new HashSet<>(); Collection conds = new HashSet<>(); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinModel.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinModel.java deleted file mode 100644 index eb06209610c0a..0000000000000 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinModel.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.query.h2.affinity.join; - -/** - * Partition join model. Describes how tables are joined with each other. - */ -public class PartitionJoinModel { -} diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java index f60ffab41f15a..e97bc0433de79 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.query.h2.affinity.join; +import org.apache.ignite.internal.util.typedef.F; import org.jetbrains.annotations.Nullable; /** @@ -35,23 +36,8 @@ public class PartitionJoinTable { /** Second affinity column name (possible when _KEY is affinity column and an alias for this column exists. */ private final String secondAffColName; - /** Whether this is not a classical table. */ - private final boolean nonTable; - - /** - * Create join table for subquery. - * - * @param alias Alias. - */ - public PartitionJoinTable(String alias) { - this.alias = alias; - - cacheName = null; - affColName = null; - secondAffColName = null; - - nonTable = true; - } + /** Join group index. */ + private int joinGrp; /** * Constructor. @@ -78,8 +64,6 @@ public PartitionJoinTable( this.affColName = affColName; this.secondAffColName = secondAffColName; } - - nonTable = false; } /** @@ -106,21 +90,38 @@ public boolean hasAffinityColumn() { /** * @return Affinity column name. */ - public String affinityColName() { + public String affinityColumnName() { return affColName; } /** * @return Second affinity column name. */ - public String secondAffinityColName() { + public String secondAffinityColumnName() { return secondAffColName; } /** - * @return Whether this is not a classical table (subquery, union, temp tables, etc). + * Check whether passed column is affinity column. + * + * @param colName Column name. + * @return {@code True} if affinity column. + */ + public boolean isAffinityColumn(String colName) { + return F.eq(colName, affColName) || F.eq(colName, secondAffColName); + } + + /** + * @return Join group index. + */ + public int joinGroup() { + return joinGrp; + } + + /** + * @param joinGrp Join group index. */ - public boolean isNonTable() { - return nonTable; + public void joinGorup(int joinGrp) { + this.joinGrp = joinGrp; } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java new file mode 100644 index 0000000000000..d8ec8faa30cbb --- /dev/null +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.h2.affinity.join; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * Partition join model. Describes how tables are joined with each other. + */ +public class PartitionTableModel { + /** All tables observed during parsing excluding outer. */ + private final Map tbls = new HashMap<>(); + + /** Join groups. */ + private final Map grps = new HashMap<>(); + + /** Talbes which are excluded from partition pruning calculation. */ + private Set excludedTblNames; + + /** Group index generator */ + private int grpIdxGen; + + /** + * Add table. + * + * @param tbl Table. + * @param aff Affinity descriptor. + */ + public void addTable(PartitionJoinTable tbl, PartitionJoinAffinityDescriptor aff) { + int grpIdx = grpIdxGen++; + + tbl.joinGorup(grpIdx); + + tbls.put(tbl.alias(), tbl); + grps.put(grpIdx, new PartitionJoinGroup(aff).addTable(tbl)); + } + + /** + * Add excluded table + * + * @param alias Alias. + */ + public void addExcludedTable(String alias) { + if (excludedTblNames == null) + excludedTblNames = new HashSet<>(); + + excludedTblNames.add(alias); + } + + /** + * Add equi-join condition. Two joined tables may possibly be merged into a single group. + * + * @param cond Condition. + */ + public void addJoin(PartitionJoinCondition cond) { + PartitionJoinTable leftTbl = tbls.get(cond.leftAlias()); + PartitionJoinTable rightTbl = tbls.get(cond.rightAlias()); + + assert leftTbl != null || (excludedTblNames != null && excludedTblNames.contains(cond.leftAlias())); + assert rightTbl != null || (excludedTblNames != null && excludedTblNames.contains(cond.rightAlias())); + + // At least one tables is excluded, return. + if (leftTbl == null || rightTbl == null) + return; + + // At least one column in condition is not affinity column, return. + if (leftTbl.isAffinityColumn(cond.leftColumn()) && rightTbl.isAffinityColumn(cond.rightColumn())) + return; + + PartitionJoinGroup leftGrp = grps.get(leftTbl.joinGroup()); + PartitionJoinGroup rightGrp = grps.get(rightTbl.joinGroup()); + + assert leftGrp != null; + assert rightGrp != null; + + // Groups are not compatible, return. + if (!leftGrp.affinityDescriptor().isCompatible(rightGrp.affinityDescriptor())) + return; + + // Safe to merge groups. + for (PartitionJoinTable tbl : rightGrp.tables()) { + tbl.joinGorup(leftTbl.joinGroup()); + + leftGrp.addTable(tbl); + } + + grps.remove(rightTbl.joinGroup()); + } +} From 608ae1b6f75cae8ed8402c6e0c1755d3b3abf573 Mon Sep 17 00:00:00 2001 From: devozerov Date: Thu, 20 Dec 2018 09:03:04 +0300 Subject: [PATCH 13/80] Preparing table model. --- .../query/h2/affinity/PartitionExtractor.java | 86 +++++-------------- .../h2/affinity/PartitionExtractorUtils.java | 71 ++++++++++----- .../affinity/join/PartitionJoinCondition.java | 31 ++++++- .../h2/affinity/join/PartitionJoinGroup.java | 12 +++ .../h2/affinity/join/PartitionTableModel.java | 11 +++ 5 files changed, 126 insertions(+), 85 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java index 07ac1114c9844..18c38a58f8713 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java @@ -21,7 +21,6 @@ import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery; import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing; import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinCondition; -import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinGroup; import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionTableModel; import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinTable; import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table; @@ -42,7 +41,6 @@ import org.jetbrains.annotations.Nullable; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -77,9 +75,10 @@ public PartitionResult extract(GridSqlQuery qry) throws IgniteCheckedException { GridSqlSelect select = (GridSqlSelect)qry; - prepareJoinModel(select.from(), select.where()); + PartitionTableModel tblModel = prepareTableModel(select.from()); // Currently we can extract data only from a single table. + // TODO: This no longer holds. GridSqlTable tbl = unwrapTable(select.from()); if (tbl == null) @@ -151,104 +150,65 @@ else if (!F.eq(desc, qryRes.descriptor())) } /** - * Prepare join model. + * Prepare table model. * * @param from FROM clause. - * @param where WHERE clause. * @return Join model. */ - private PartitionTableModel prepareJoinModel(GridSqlAst from, GridSqlAst where) { - Set leftJoined = new HashSet<>(); - Collection conds = new HashSet<>(); + private PartitionTableModel prepareTableModel(GridSqlAst from) { + PartitionTableModel res = new PartitionTableModel(); - Collection grps = prepareJoinModelTables(from, leftJoined, conds); + prepareTableModel0(from, res); - // TODO - return null; - } - - /** - * Prepare expressions from WHERE condition. - *

- * All conditions which qualifies for joined tables must be located at the top level of conjunctive expression. - * That is, (joinCond AND ...) is OK for us. But (joinCond OR ...) isn't. - *

- * Also we do not expect any additional join expressions on LEFT JOINed columns, so if they are met, processing is - * stopped. - * - * @param where WHERE condition. - * @param leftJoined Left joined tables. - * @param conds Condirtions. - */ - private boolean prepareJoinModelConditions(GridSqlAst where, Collection leftJoined, - Collection conds) { - // TODO - return false; + return res; } /** * Prepare tables which will be used in join model. * * @param from From flag. - * @param conds Conditions. + * @param model Table model. * @return {@code True} if extracted tables successfully, {@code false} if failed to extract. */ - private List prepareJoinModelTables( - GridSqlAst from, - Collection leftJoined, - Collection conds - ) { + private List prepareTableModel0(GridSqlAst from, PartitionTableModel model) { if (from instanceof GridSqlJoin) { // Process JOIN recursively. GridSqlJoin join = (GridSqlJoin)from; - List leftGrp = prepareJoinModelTables(join.leftTable(), leftJoined, conds); - - if (leftGrp == null) - return null; - - List rightGrp = prepareJoinModelTables(join.leftTable(), leftJoined, conds); - - if (rightGrp == null) - return null; + List leftTbls = prepareTableModel0(join.leftTable(), model); + List rightTbls = prepareTableModel0(join.leftTable(), model); if (join.isLeftOuter()) { - // Do not support complex LEFT-RIGHT join variations for now. - if (rightGrp.size() != 1) - return null; - // "a LEFT JOIN b" is transformed into "a", and "b" is put into special stop-list. // If a condition is met on "b" afterwards, we will stop partition pruning process. - for (PartitionJoinTable rightTbl : rightGrp.get(0).tables()) - leftJoined.add(rightTbl.alias()); + for (PartitionJoinTable rightTbl : rightTbls) + model.addExcludedTable(rightTbl.alias()); - return leftGrp; + return leftTbls; } - // Extract condition from JOIN. Only equijoins are supported for now. + // Extract equi-join or cross-join from condition. For normal INNER JOINs most likely we will have "1=1" + // cross join here, real join condition will be found in WHERE clause later. GridSqlElement on = join.on(); if (!PartitionExtractorUtils.isCrossJoinCondition(on)) { - PartitionJoinCondition cond = PartitionExtractorUtils.tryParseEquiJoinCondition(on); + PartitionJoinCondition cond = PartitionExtractorUtils.parseJoinCondition(on); if (cond != null) - conds.add(cond); - else - // Non equi-join, stop partition pruning. - return null; + model.addJoin(cond); } - ArrayList res = new ArrayList<>(leftGrp.size() + rightGrp.size()); + ArrayList res = new ArrayList<>(leftTbls.size() + rightTbls.size()); - res.addAll(leftGrp); - res.addAll(rightGrp); + res.addAll(leftTbls); + res.addAll(rightTbls); return res; } - PartitionJoinGroup grp = PartitionExtractorUtils.joinGroupForTable(from); + PartitionJoinTable tbl = PartitionExtractorUtils.prepareTable(from, model); - return Collections.singletonList(grp); + return Collections.singletonList(tbl); } /** diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java index 1aa8605ab403f..19be6b7891627 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java @@ -22,8 +22,8 @@ import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionAffinityFunctionType; import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinAffinityDescriptor; import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinCondition; -import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinGroup; import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinTable; +import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionTableModel; import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlAlias; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlAst; @@ -34,7 +34,6 @@ import org.apache.ignite.internal.processors.query.h2.sql.GridSqlOperationType; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlTable; import org.h2.table.Column; -import org.h2.value.Value; /** * Utility methods for partition extraction. @@ -42,12 +41,13 @@ public class PartitionExtractorUtils { /** - * Prepare join group for a single table. + * Prepare single table. * - * @param from Table. - * @return Join group. + * @param from Expression. + * @param tblModel Table model. + * @return Added table or {@code null} if table is exlcuded from the model. */ - public static PartitionJoinGroup joinGroupForTable(GridSqlAst from) { + public static PartitionJoinTable prepareTable(GridSqlAst from, PartitionTableModel tblModel) { String alias = null; if (from instanceof GridSqlAlias) { @@ -62,11 +62,15 @@ public static PartitionJoinGroup joinGroupForTable(GridSqlAst from) { GridH2Table tbl0 = from0.dataTable(); - if (tbl0 == null) - // Unknown table type, e.g. temp table. - return new PartitionJoinGroup(null).addTable(new PartitionJoinTable(alias)); + // Unknown table type, e.g. temp table. + if (tbl0 == null) { + tblModel.addExcludedTable(alias); + + return null; + } // Use identifier string because there might be two table with the same name but form different schemas. + // TODO: Be very careful here. Seems that alias should never be null here! if (alias == null) alias = tbl0.identifierString(); @@ -88,17 +92,20 @@ public static PartitionJoinGroup joinGroupForTable(GridSqlAst from) { } } - PartitionJoinTable joinTbl = new PartitionJoinTable(alias, cacheName, affColName, secondAffColName); + PartitionJoinTable tbl = new PartitionJoinTable(alias, cacheName, affColName, secondAffColName); + PartitionJoinAffinityDescriptor aff = affinityDescriptorForCache(tbl0.cacheInfo().config()); - PartitionJoinAffinityDescriptor affDesc = affinityDescriptorForCache(tbl0.cacheInfo().config()); + tblModel.addTable(tbl, aff); - return new PartitionJoinGroup(affDesc).addTable(joinTbl); + return tbl; } else { - // Subquery/union + // Subquery/union/view, etc. assert alias != null; - return new PartitionJoinGroup(null).addTable(new PartitionJoinTable(alias)); + tblModel.addExcludedTable(alias); + + return null; } } @@ -113,15 +120,15 @@ public static boolean isCrossJoinCondition(GridSqlAst on) { GridSqlOperation on0 = (GridSqlOperation)on; if (on0.operationType() == GridSqlOperationType.EQUAL) { - GridSqlConst left = PartitionExtractor.unwrapConst(on0.child(0)); - GridSqlConst right = PartitionExtractor.unwrapConst(on0.child(1)); + GridSqlConst leftConst = PartitionExtractor.unwrapConst(on0.child(0)); + GridSqlConst rightConst = PartitionExtractor.unwrapConst(on0.child(1)); - if (left != null && right != null) { + if (leftConst != null && rightConst != null) { try { - int leftVal = left.value().getInt(); - int rightVal = right.value().getInt(); + int leftConstval = leftConst.value().getInt(); + int rightConstVal = rightConst.value().getInt(); - return leftVal == rightVal; + return leftConstval == rightConstVal; } catch (Exception ignore) { // No-op. @@ -140,11 +147,33 @@ public static boolean isCrossJoinCondition(GridSqlAst on) { * @param on Initial AST. * @return Join condition or {@code null} if not simple equijoin. */ - public static PartitionJoinCondition tryParseEquiJoinCondition(GridSqlElement on) { + public static PartitionJoinCondition parseJoinCondition(GridSqlElement on) { if (on instanceof GridSqlOperation) { GridSqlOperation on0 = (GridSqlOperation)on; if (on0.operationType() == GridSqlOperationType.EQUAL) { + // Check for cross-join first. + GridSqlConst leftConst = PartitionExtractor.unwrapConst(on0.child(0)); + GridSqlConst rightConst = PartitionExtractor.unwrapConst(on0.child(1)); + + if (leftConst != null && rightConst != null) { + try { + int leftConstval = leftConst.value().getInt(); + int rightConstVal = rightConst.value().getInt(); + + if (leftConstval == rightConstVal) + return PartitionJoinCondition.CROSS; + } + catch (Exception ignore) { + // No-op. + } + } + + // This is not cross-join, neither normal join between columns. + if (leftConst != null || rightConst != null) + return null; + + // Check for normal equi-join. GridSqlColumn left = PartitionExtractor.unwrapColumn(on0.child(0)); GridSqlColumn right = PartitionExtractor.unwrapColumn(on0.child(1)); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinCondition.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinCondition.java index 0975452883685..4f5279c47188a 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinCondition.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinCondition.java @@ -23,6 +23,9 @@ * Join condition. */ public class PartitionJoinCondition { + /** Cross JOIN. */ + public static final PartitionJoinCondition CROSS = new PartitionJoinCondition(null, null, null, null, true); + /** Left alias. */ private final String leftAlias; @@ -35,6 +38,9 @@ public class PartitionJoinCondition { /** Right column name. */ private final String rightCol; + /** Whether this is a cross-join. */ + private final boolean cross; + /** * Constructor. * @@ -44,10 +50,25 @@ public class PartitionJoinCondition { * @param rightCol Right column name. */ public PartitionJoinCondition(String leftAlias, String rightAlias, String leftCol, String rightCol) { + this(leftAlias, rightAlias, leftCol, rightCol, false); + } + + /** + * Constructor. + * + * @param leftAlias Left alias. + * @param rightAlias Right alias. + * @param leftCol Left column name. + * @param rightCol Right column name. + * @param cross Whether this is a cross-join. + */ + private PartitionJoinCondition(String leftAlias, String rightAlias, String leftCol, String rightCol, + boolean cross) { this.leftAlias = leftAlias; this.rightAlias = rightAlias; this.leftCol = leftCol; this.rightCol = rightCol; + this.cross = cross; } /** @@ -78,6 +99,13 @@ public String rightColumn() { return rightCol; } + /** + * @return Wheter this is a cross-join. + */ + public boolean cross() { + return cross; + } + /** {@inheritDoc} */ @Override public int hashCode() { int res = leftAlias.hashCode(); @@ -85,6 +113,7 @@ public String rightColumn() { res = 31 * res + rightAlias.hashCode(); res = 31 * res + leftCol.hashCode(); res = 31 * res + rightCol.hashCode(); + res = 31 * res + Boolean.hashCode(cross); return res; } @@ -95,7 +124,7 @@ public String rightColumn() { PartitionJoinCondition other = (PartitionJoinCondition)obj; return F.eq(leftAlias, other.leftAlias) && F.eq(rightAlias, other.rightAlias) && - F.eq(leftCol, other.leftCol) && F.eq(rightCol, other.rightCol); + F.eq(leftCol, other.leftCol) && F.eq(rightCol, other.rightCol) && F.eq(cross, other.cross); } return false; diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java index 09d6d8e50ac23..bf6a6ae365bc5 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java @@ -60,6 +60,18 @@ public PartitionJoinGroup addTable(PartitionJoinTable tbl) { return this; } + /** + * Remove table from the group. + * + * @param tbl Table. + * @return If group is empty after removal. + */ + public boolean removeTable(PartitionJoinTable tbl) { + tbls.remove(tbl); + + return tbls.isEmpty(); + } + /** * @return Affinity descriptor. */ diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java index d8ec8faa30cbb..db9a78e3da507 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java @@ -59,6 +59,17 @@ public void addTable(PartitionJoinTable tbl, PartitionJoinAffinityDescriptor aff * @param alias Alias. */ public void addExcludedTable(String alias) { + PartitionJoinTable tbl = tbls.remove(alias); + + if (tbl != null) { + PartitionJoinGroup grp = grps.get(tbl.joinGroup()); + + assert grp != null; + + if (grp.removeTable(tbl)) + grps.remove(tbl.joinGroup()); + } + if (excludedTblNames == null) excludedTblNames = new HashSet<>(); From 6e40856de0dd4124b6b6a04395df901ebaa96fba Mon Sep 17 00:00:00 2001 From: devozerov Date: Thu, 20 Dec 2018 12:06:46 +0300 Subject: [PATCH 14/80] WIP. --- .../h2/affinity/PartitionExtractorUtils.java | 28 +++++++++------- .../join/PartitionJoinAffinityDescriptor.java | 32 +++++++------------ 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java index 19be6b7891627..2cd4d1822ab33 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.query.h2.affinity; +import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionAffinityFunctionType; @@ -48,13 +49,13 @@ public class PartitionExtractorUtils { * @return Added table or {@code null} if table is exlcuded from the model. */ public static PartitionJoinTable prepareTable(GridSqlAst from, PartitionTableModel tblModel) { - String alias = null; + // TODO: Dangerous, but should be true. + // Unwrap alias. We assume that every table must be aliased. + assert from instanceof GridSqlAlias; - if (from instanceof GridSqlAlias) { - alias = ((GridSqlAlias)from).alias(); + String alias = ((GridSqlAlias)from).alias(); - from = from.child(); - } + from = from.child(); if (from instanceof GridSqlTable) { // Normal table. @@ -69,11 +70,6 @@ public static PartitionJoinTable prepareTable(GridSqlAst from, PartitionTableMod return null; } - // Use identifier string because there might be two table with the same name but form different schemas. - // TODO: Be very careful here. Seems that alias should never be null here! - if (alias == null) - alias = tbl0.identifierString(); - String cacheName = tbl0.cacheName(); String affColName = null; @@ -95,6 +91,13 @@ public static PartitionJoinTable prepareTable(GridSqlAst from, PartitionTableMod PartitionJoinTable tbl = new PartitionJoinTable(alias, cacheName, affColName, secondAffColName); PartitionJoinAffinityDescriptor aff = affinityDescriptorForCache(tbl0.cacheInfo().config()); + if (aff == null) { + // Non-standard affinity, exclude table. + tblModel.addExcludedTable(alias); + + return null; + } + tblModel.addTable(tbl, aff); return tbl; @@ -199,11 +202,14 @@ public static PartitionJoinCondition parseJoinCondition(GridSqlElement on) { * @return Affinity identifier. */ private static PartitionJoinAffinityDescriptor affinityDescriptorForCache(CacheConfiguration ccfg) { + // Partition could be extracted only from PARTITIONED cache. + if (ccfg.getCacheMode() != CacheMode.PARTITIONED) + return null; + PartitionAffinityFunctionType aff = ccfg.getAffinity().getClass().equals(RendezvousAffinityFunction.class) ? PartitionAffinityFunctionType.RENDEZVOUS : PartitionAffinityFunctionType.CUSTOM; return new PartitionJoinAffinityDescriptor( - ccfg.getCacheMode(), aff, ccfg.getAffinity().partitions(), ccfg.getNodeFilter() != null diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinAffinityDescriptor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinAffinityDescriptor.java index 108535c0bf02e..62886023f15a6 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinAffinityDescriptor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinAffinityDescriptor.java @@ -29,9 +29,6 @@ public class PartitionJoinAffinityDescriptor implements Serializable { /** */ private static final long serialVersionUID = 0L; - /** Cache mode. */ - private final CacheMode cacheMode; - /** Affinity function type. */ private final PartitionAffinityFunctionType affFunc; @@ -44,18 +41,15 @@ public class PartitionJoinAffinityDescriptor implements Serializable { /** * Constructor. * - * @param cacheMode Cache mode. * @param affFunc Affinity function type. * @param parts Number of partitions. * @param hasNodeFilter Whether node filter is set. */ public PartitionJoinAffinityDescriptor( - CacheMode cacheMode, PartitionAffinityFunctionType affFunc, int parts, boolean hasNodeFilter ) { - this.cacheMode = cacheMode; this.affFunc = affFunc; this.parts = parts; this.hasNodeFilter = hasNodeFilter; @@ -68,21 +62,17 @@ public PartitionJoinAffinityDescriptor( * @return {@code True} if compatible. */ public boolean isCompatible(PartitionJoinAffinityDescriptor other) { - // REPLICATED caches has special treatment during parititon pruning, so exclude them. - if (cacheMode == CacheMode.PARTITIONED) { - // Rendezvous affinity function is deterministic and doesn't depend on previous cluster view changes. - // In future other user affinity functions would be applicable as well if explicityl marked deterministic. - if (affFunc == PartitionAffinityFunctionType.RENDEZVOUS) { - // We cannot be sure that two caches are co-located if custom node filter is present. - // Nota that technically we may try to compare two filters. However, this adds unnecessary complexity - // and potential deserialization issues when SQL is called from client nodes or thin clients. - if (!hasNodeFilter) { - return - other.cacheMode == CacheMode.PARTITIONED && - other.affFunc == PartitionAffinityFunctionType.RENDEZVOUS && - !other.hasNodeFilter && - other.parts == parts; - } + // Rendezvous affinity function is deterministic and doesn't depend on previous cluster view changes. + // In future other user affinity functions would be applicable as well if explicityl marked deterministic. + if (affFunc == PartitionAffinityFunctionType.RENDEZVOUS) { + // We cannot be sure that two caches are co-located if custom node filter is present. + // Nota that technically we may try to compare two filters. However, this adds unnecessary complexity + // and potential deserialization issues when SQL is called from client nodes or thin clients. + if (!hasNodeFilter) { + return + other.affFunc == PartitionAffinityFunctionType.RENDEZVOUS && + !other.hasNodeFilter && + other.parts == parts; } } From 04a5f5eaabda51ba53f6495090709a945b532a33 Mon Sep 17 00:00:00 2001 From: devozerov Date: Thu, 20 Dec 2018 12:08:26 +0300 Subject: [PATCH 15/80] WIP. --- .../processors/query/h2/affinity/PartitionExtractor.java | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java index 18c38a58f8713..36eaf2fdf893d 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java @@ -75,6 +75,7 @@ public PartitionResult extract(GridSqlQuery qry) throws IgniteCheckedException { GridSqlSelect select = (GridSqlSelect)qry; + // Prepare table model. PartitionTableModel tblModel = prepareTableModel(select.from()); // Currently we can extract data only from a single table. From 276b4afd153307a7940ef4947fd7eb0aa2ed1914 Mon Sep 17 00:00:00 2001 From: devozerov Date: Thu, 20 Dec 2018 19:31:37 +0300 Subject: [PATCH 16/80] Recursive disjunction. --- .../query/h2/affinity/PartitionExtractor.java | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java index 36eaf2fdf893d..51467f8be26b1 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java @@ -86,7 +86,7 @@ public PartitionResult extract(GridSqlQuery qry) throws IgniteCheckedException { return null; // Do extract. - PartitionNode tree = extractFromExpression(select.where()); + PartitionNode tree = extractFromExpression(select.where(), false); assert tree != null; @@ -232,10 +232,14 @@ private List prepareTableModel0(GridSqlAst from, PartitionTa * Extract partitions from expression. * * @param expr Expression. + * @param disjunct Whether current processing frame is located under disjunction ("OR"). In this case we cannot + * rely on join expressions like (A.a = B.b) to build co-location model because another conflicting + * join expression on the same tables migth be located on the other side of the "OR". + * Example: "JOIN on A.a = B.b OR A.a > B.b". * @return Partition tree. */ @SuppressWarnings("EnumSwitchStatementWhichMissesCases") - private PartitionNode extractFromExpression(GridSqlAst expr) throws IgniteCheckedException { + private PartitionNode extractFromExpression(GridSqlAst expr, boolean disjunct) throws IgniteCheckedException { PartitionNode res = PartitionAllNode.INSTANCE; if (expr instanceof GridSqlOperation) { @@ -243,7 +247,7 @@ private PartitionNode extractFromExpression(GridSqlAst expr) throws IgniteChecke switch (op.operationType()) { case AND: - res = extractFromAnd(op); + res = extractFromAnd(op, disjunct); break; @@ -270,13 +274,14 @@ private PartitionNode extractFromExpression(GridSqlAst expr) throws IgniteChecke * Extract partition information from AND. * * @param op Operation. + * @param disjunct Disjunction marker. * @return Partition. */ - private PartitionNode extractFromAnd(GridSqlOperation op) throws IgniteCheckedException { + private PartitionNode extractFromAnd(GridSqlOperation op, boolean disjunct) throws IgniteCheckedException { assert op.size() == 2; - PartitionNode part1 = extractFromExpression(op.child(0)); - PartitionNode part2 = extractFromExpression(op.child(1)); + PartitionNode part1 = extractFromExpression(op.child(0), disjunct); + PartitionNode part2 = extractFromExpression(op.child(1), disjunct); return new PartitionCompositeNode(part1, part2, PartitionCompositeNodeOperator.AND); } @@ -290,8 +295,9 @@ private PartitionNode extractFromAnd(GridSqlOperation op) throws IgniteCheckedEx private PartitionNode extractFromOr(GridSqlOperation op) throws IgniteCheckedException { assert op.size() == 2; - PartitionNode part1 = extractFromExpression(op.child(0)); - PartitionNode part2 = extractFromExpression(op.child(1)); + // Parse inner expressions recursively with disjuncion flag set. + PartitionNode part1 = extractFromExpression(op.child(0), true); + PartitionNode part2 = extractFromExpression(op.child(1), true); return new PartitionCompositeNode(part1, part2, PartitionCompositeNodeOperator.OR); } From 0172b894dd6e16b66fce1ea74149a4b4a35b57e5 Mon Sep 17 00:00:00 2001 From: devozerov Date: Thu, 20 Dec 2018 19:36:37 +0300 Subject: [PATCH 17/80] Pass table model recursively. --- .../query/h2/affinity/PartitionExtractor.java | 53 ++++++++++++------- 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java index 51467f8be26b1..bd5de385a4d87 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java @@ -86,7 +86,7 @@ public PartitionResult extract(GridSqlQuery qry) throws IgniteCheckedException { return null; // Do extract. - PartitionNode tree = extractFromExpression(select.where(), false); + PartitionNode tree = extractFromExpression(select.where(), tblModel, false); assert tree != null; @@ -232,6 +232,7 @@ private List prepareTableModel0(GridSqlAst from, PartitionTa * Extract partitions from expression. * * @param expr Expression. + * @param tblModel Table model. * @param disjunct Whether current processing frame is located under disjunction ("OR"). In this case we cannot * rely on join expressions like (A.a = B.b) to build co-location model because another conflicting * join expression on the same tables migth be located on the other side of the "OR". @@ -239,7 +240,8 @@ private List prepareTableModel0(GridSqlAst from, PartitionTa * @return Partition tree. */ @SuppressWarnings("EnumSwitchStatementWhichMissesCases") - private PartitionNode extractFromExpression(GridSqlAst expr, boolean disjunct) throws IgniteCheckedException { + private PartitionNode extractFromExpression(GridSqlAst expr, PartitionTableModel tblModel, boolean disjunct) + throws IgniteCheckedException { PartitionNode res = PartitionAllNode.INSTANCE; if (expr instanceof GridSqlOperation) { @@ -247,22 +249,22 @@ private PartitionNode extractFromExpression(GridSqlAst expr, boolean disjunct) t switch (op.operationType()) { case AND: - res = extractFromAnd(op, disjunct); + res = extractFromAnd(op, tblModel, disjunct); break; case OR: - res = extractFromOr(op); + res = extractFromOr(op, tblModel); break; case IN: - res = extractFromIn(op); + res = extractFromIn(op, tblModel); break; case EQUAL: - res = extractFromEqual(op); + res = extractFromEqual(op, tblModel); } } @@ -274,14 +276,16 @@ private PartitionNode extractFromExpression(GridSqlAst expr, boolean disjunct) t * Extract partition information from AND. * * @param op Operation. + * @param tblModel Table model. * @param disjunct Disjunction marker. * @return Partition. */ - private PartitionNode extractFromAnd(GridSqlOperation op, boolean disjunct) throws IgniteCheckedException { + private PartitionNode extractFromAnd(GridSqlOperation op, PartitionTableModel tblModel, boolean disjunct) + throws IgniteCheckedException { assert op.size() == 2; - PartitionNode part1 = extractFromExpression(op.child(0), disjunct); - PartitionNode part2 = extractFromExpression(op.child(1), disjunct); + PartitionNode part1 = extractFromExpression(op.child(0), tblModel, disjunct); + PartitionNode part2 = extractFromExpression(op.child(1), tblModel, disjunct); return new PartitionCompositeNode(part1, part2, PartitionCompositeNodeOperator.AND); } @@ -290,14 +294,16 @@ private PartitionNode extractFromAnd(GridSqlOperation op, boolean disjunct) thro * Extract partition information from OR. * * @param op Operation. + * @param tblModel Table model. * @return Partition. */ - private PartitionNode extractFromOr(GridSqlOperation op) throws IgniteCheckedException { + private PartitionNode extractFromOr(GridSqlOperation op, PartitionTableModel tblModel) + throws IgniteCheckedException { assert op.size() == 2; // Parse inner expressions recursively with disjuncion flag set. - PartitionNode part1 = extractFromExpression(op.child(0), true); - PartitionNode part2 = extractFromExpression(op.child(1), true); + PartitionNode part1 = extractFromExpression(op.child(0), tblModel, true); + PartitionNode part2 = extractFromExpression(op.child(1), tblModel, true); return new PartitionCompositeNode(part1, part2, PartitionCompositeNodeOperator.OR); } @@ -306,9 +312,11 @@ private PartitionNode extractFromOr(GridSqlOperation op) throws IgniteCheckedExc * Extract partition information from IN. * * @param op Operation. + * @param tblModel Table model. * @return Partition. */ - private PartitionNode extractFromIn(GridSqlOperation op) throws IgniteCheckedException { + private PartitionNode extractFromIn(GridSqlOperation op, PartitionTableModel tblModel) + throws IgniteCheckedException { // Operation should contain at least two children: left (column) and right (const or column). if (op.size() < 2) return PartitionAllNode.INSTANCE; @@ -348,7 +356,7 @@ else if (right instanceof GridSqlParameter) { return PartitionAllNode.INSTANCE; // Extract. - PartitionSingleNode part = extractSingle(leftCol.column(), rightConst, rightParam); + PartitionSingleNode part = extractSingle(leftCol.column(), rightConst, rightParam, tblModel); // Same thing as above: single unknown partition in disjunction defeats optimization. if (part == null) @@ -364,9 +372,11 @@ else if (right instanceof GridSqlParameter) { * Extract partition information from equality. * * @param op Operation. + * @param tblModel Table model. * @return Partition. */ - private PartitionNode extractFromEqual(GridSqlOperation op) throws IgniteCheckedException { + private PartitionNode extractFromEqual(GridSqlOperation op, PartitionTableModel tblModel) + throws IgniteCheckedException { assert op.operationType() == GridSqlOperationType.EQUAL; GridSqlElement left = op.child(0); @@ -394,7 +404,7 @@ else if (right instanceof GridSqlParameter) { else return PartitionAllNode.INSTANCE; - PartitionSingleNode part = extractSingle(leftCol.column(), rightConst, rightParam); + PartitionSingleNode part = extractSingle(leftCol.column(), rightConst, rightParam, tblModel); return part != null ? part : PartitionAllNode.INSTANCE; } @@ -405,10 +415,16 @@ else if (right instanceof GridSqlParameter) { * @param leftCol Left column. * @param rightConst Right constant. * @param rightParam Right parameter. + * @param tblModel Table model. * @return Partition or {@code null} if failed to extract. */ - @Nullable private PartitionSingleNode extractSingle(Column leftCol, GridSqlConst rightConst, - GridSqlParameter rightParam) throws IgniteCheckedException { + // TODO: Make sure that join equality is processed on upper levels. + @Nullable private PartitionSingleNode extractSingle( + Column leftCol, + GridSqlConst rightConst, + GridSqlParameter rightParam, + PartitionTableModel tblModel + ) throws IgniteCheckedException { assert leftCol != null; assert leftCol.getTable() != null; assert leftCol.getTable() instanceof GridH2Table; @@ -418,6 +434,7 @@ else if (right instanceof GridSqlParameter) { if (!tbl.isColumnForPartitionPruning(leftCol)) return null; + // TODO: Use table model instead PartitionTableDescriptor tblDesc = descriptor(tbl); if (rightConst != null) { From 7a4ce2b9b21c6fafd40eac1805ee39cca8f3d5b3 Mon Sep 17 00:00:00 2001 From: devozerov Date: Thu, 20 Dec 2018 19:44:52 +0300 Subject: [PATCH 18/80] Add join conditions to the model. --- .../query/h2/affinity/PartitionExtractor.java | 26 +++++++++------ .../h2/affinity/PartitionExtractorUtils.java | 32 ------------------- 2 files changed, 16 insertions(+), 42 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java index bd5de385a4d87..39d5edae80f42 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java @@ -190,14 +190,10 @@ private List prepareTableModel0(GridSqlAst from, PartitionTa // Extract equi-join or cross-join from condition. For normal INNER JOINs most likely we will have "1=1" // cross join here, real join condition will be found in WHERE clause later. - GridSqlElement on = join.on(); + PartitionJoinCondition cond = PartitionExtractorUtils.parseJoinCondition(join.on()); - if (!PartitionExtractorUtils.isCrossJoinCondition(on)) { - PartitionJoinCondition cond = PartitionExtractorUtils.parseJoinCondition(on); - - if (cond != null) - model.addJoin(cond); - } + if (cond != null && cond.cross()) + model.addJoin(cond); ArrayList res = new ArrayList<>(leftTbls.size() + rightTbls.size()); @@ -264,7 +260,7 @@ private PartitionNode extractFromExpression(GridSqlAst expr, PartitionTableModel break; case EQUAL: - res = extractFromEqual(op, tblModel); + res = extractFromEqual(op, tblModel, disjunct); } } @@ -373,9 +369,10 @@ else if (right instanceof GridSqlParameter) { * * @param op Operation. * @param tblModel Table model. + * @param disjunct Disjunction flag. When set possible join expression will not be processed. * @return Partition. */ - private PartitionNode extractFromEqual(GridSqlOperation op, PartitionTableModel tblModel) + private PartitionNode extractFromEqual(GridSqlOperation op, PartitionTableModel tblModel, boolean disjunct) throws IgniteCheckedException { assert op.operationType() == GridSqlOperationType.EQUAL; @@ -401,6 +398,16 @@ else if (right instanceof GridSqlParameter) { rightConst = null; rightParam = (GridSqlParameter)right; } + else if (right instanceof GridSqlColumn) { + if (!disjunct) { + PartitionJoinCondition cond = PartitionExtractorUtils.parseJoinCondition(op); + + if (cond != null && cond.cross()) + tblModel.addJoin(cond); + } + + return PartitionAllNode.INSTANCE; + } else return PartitionAllNode.INSTANCE; @@ -418,7 +425,6 @@ else if (right instanceof GridSqlParameter) { * @param tblModel Table model. * @return Partition or {@code null} if failed to extract. */ - // TODO: Make sure that join equality is processed on upper levels. @Nullable private PartitionSingleNode extractSingle( Column leftCol, GridSqlConst rightConst, diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java index 2cd4d1822ab33..4d9128682ba0b 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java @@ -49,7 +49,6 @@ public class PartitionExtractorUtils { * @return Added table or {@code null} if table is exlcuded from the model. */ public static PartitionJoinTable prepareTable(GridSqlAst from, PartitionTableModel tblModel) { - // TODO: Dangerous, but should be true. // Unwrap alias. We assume that every table must be aliased. assert from instanceof GridSqlAlias; @@ -112,37 +111,6 @@ public static PartitionJoinTable prepareTable(GridSqlAst from, PartitionTableMod } } - /** - * Check whether this is a cross-join condition, i.e. 1=1. - * - * @param on Condition. - * @return {@code True} if cross-join. - */ - public static boolean isCrossJoinCondition(GridSqlAst on) { - if (on instanceof GridSqlOperation) { - GridSqlOperation on0 = (GridSqlOperation)on; - - if (on0.operationType() == GridSqlOperationType.EQUAL) { - GridSqlConst leftConst = PartitionExtractor.unwrapConst(on0.child(0)); - GridSqlConst rightConst = PartitionExtractor.unwrapConst(on0.child(1)); - - if (leftConst != null && rightConst != null) { - try { - int leftConstval = leftConst.value().getInt(); - int rightConstVal = rightConst.value().getInt(); - - return leftConstval == rightConstVal; - } - catch (Exception ignore) { - // No-op. - } - } - } - } - - return false; - } - /** * Try parsing condition as simple JOIN codition. Only equijoins are supported for now, so anything more complex * than "A.a = B.b" are not processed. From 24ea3232c44cf6f733fd8e9574203028b0400c32 Mon Sep 17 00:00:00 2001 From: devozerov Date: Thu, 20 Dec 2018 20:47:40 +0300 Subject: [PATCH 19/80] Added join groups to nodes. --- .../query/h2/affinity/PartitionAllNode.java | 6 ++++ .../h2/affinity/PartitionCompositeNode.java | 21 ++++++++++++-- .../h2/affinity/PartitionConstantNode.java | 7 +++-- .../query/h2/affinity/PartitionExtractor.java | 28 +++++++++++-------- .../query/h2/affinity/PartitionGroupNode.java | 7 +++++ .../query/h2/affinity/PartitionNode.java | 5 ++++ .../query/h2/affinity/PartitionNoneNode.java | 6 ++++ .../h2/affinity/PartitionParameterNode.java | 3 +- .../h2/affinity/PartitionSingleNode.java | 15 +++++++--- .../h2/affinity/join/PartitionJoinTable.java | 6 ++++ .../h2/affinity/join/PartitionTableModel.java | 19 +++++++++++++ 11 files changed, 101 insertions(+), 22 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionAllNode.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionAllNode.java index 842d82c76991e..f8110182e7aa5 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionAllNode.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionAllNode.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.query.h2.affinity; +import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionTableModel; import org.apache.ignite.internal.util.typedef.internal.S; import java.util.Collection; @@ -40,6 +41,11 @@ private PartitionAllNode() { return null; } + /** {@inheritDoc} */ + @Override public int joinGroup() { + return PartitionTableModel.GRP_NONE; + } + /** {@inheritDoc} */ @Override public String toString() { return S.toString(PartitionAllNode.class, this); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionCompositeNode.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionCompositeNode.java index 2cb330f342be2..affaa3fbfc1d5 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionCompositeNode.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionCompositeNode.java @@ -83,6 +83,12 @@ else if (rightParts == null) return leftParts; } + /** {@inheritDoc} */ + @Override public int joinGroup() { + // Similar to composite node, we cannot cache join group value here. + return left.joinGroup(); + } + /** {@inheritDoc} */ @Override public PartitionNode optimize() { PartitionNode left = this.left; @@ -103,9 +109,13 @@ else if (rightParts == null) return optimizeSpecial(right, left); // If one of child nodes cannot be optimized, nothing can be done further. - // Note that we cannot return "this" here because left or right parts might have been optimized. - if (left instanceof PartitionCompositeNode || right instanceof PartitionCompositeNode) + // Note that we cannot return "this" here because left or right parts might have been changed. + if (left instanceof PartitionCompositeNode || right instanceof PartitionCompositeNode) { + // TODO: Check co-location groups here. + // TODO: If they do not match for OR - return "ALL" + // TODO: If they do not match for AND - then what? return new PartitionCompositeNode(left, right, op); + } // Try optimizing composite nodes. if (left instanceof PartitionGroupNode) @@ -180,6 +190,7 @@ private PartitionNode optimizeGroup(PartitionGroupNode left, PartitionNode right * @return Optimized node. */ private PartitionNode optimizeGroupAnd(PartitionGroupNode left, PartitionNode right) { + // TODO: Careful assert op == PartitionCompositeNodeOperator.AND; // Optimistic check whether both sides are equal. @@ -236,6 +247,7 @@ else if (consts.size() == 1) * @return Optimized node. */ private PartitionNode optimizeGroupOr(PartitionGroupNode left, PartitionNode right) { + // TODO: Careful assert op == PartitionCompositeNodeOperator.OR; HashSet siblings = new HashSet<>(left.siblings()); @@ -289,7 +301,9 @@ private PartitionNode optimizeSimpleAnd(PartitionSingleNode left, PartitionSingl // X and Y -> NONE return PartitionNoneNode.INSTANCE; - // Otherwise it is a mixed set, cannot reduce. + // TODO: What should be done here wrt/ join groups? + + // Otherwise this is either a mixed set, cannot reduce. // X and :Y -> (X) AND (:Y) return new PartitionCompositeNode(left, right, PartitionCompositeNodeOperator.AND); } @@ -302,6 +316,7 @@ private PartitionNode optimizeSimpleAnd(PartitionSingleNode left, PartitionSingl * @return Optimized node. */ private PartitionNode optimizeSimpleOr(PartitionSingleNode left, PartitionSingleNode right) { + // TODO: Merge only if they belong to the same group, otherwise this is "ALL" assert op == PartitionCompositeNodeOperator.OR; return left.equals(right) ? left : PartitionGroupNode.merge(left, right); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionConstantNode.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionConstantNode.java index 9efafe4536999..aa04e4ee747f8 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionConstantNode.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionConstantNode.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.query.h2.affinity; +import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinTable; import org.apache.ignite.internal.util.typedef.internal.S; /** @@ -29,11 +30,11 @@ public class PartitionConstantNode extends PartitionSingleNode { /** * Constructor. * - * @param resolver Resolver. + * @param tbl Table. * @param part Partition. */ - public PartitionConstantNode(PartitionTableDescriptor resolver, int part) { - super(resolver); + public PartitionConstantNode(PartitionJoinTable tbl, int part) { + super(tbl); this.part = part; } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java index 39d5edae80f42..b2d2ae655b862 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java @@ -352,7 +352,7 @@ else if (right instanceof GridSqlParameter) { return PartitionAllNode.INSTANCE; // Extract. - PartitionSingleNode part = extractSingle(leftCol.column(), rightConst, rightParam, tblModel); + PartitionSingleNode part = extractSingle(leftCol, rightConst, rightParam, tblModel); // Same thing as above: single unknown partition in disjunction defeats optimization. if (part == null) @@ -411,7 +411,7 @@ else if (right instanceof GridSqlColumn) { else return PartitionAllNode.INSTANCE; - PartitionSingleNode part = extractSingle(leftCol.column(), rightConst, rightParam, tblModel); + PartitionSingleNode part = extractSingle(leftCol, rightConst, rightParam, tblModel); return part != null ? part : PartitionAllNode.INSTANCE; } @@ -426,30 +426,36 @@ else if (right instanceof GridSqlColumn) { * @return Partition or {@code null} if failed to extract. */ @Nullable private PartitionSingleNode extractSingle( - Column leftCol, + GridSqlColumn leftCol, GridSqlConst rightConst, GridSqlParameter rightParam, PartitionTableModel tblModel ) throws IgniteCheckedException { assert leftCol != null; - assert leftCol.getTable() != null; - assert leftCol.getTable() instanceof GridH2Table; - GridH2Table tbl = (GridH2Table)leftCol.getTable(); + Column leftCol0 = leftCol.column(); - if (!tbl.isColumnForPartitionPruning(leftCol)) + assert leftCol0.getTable() != null; + assert leftCol0.getTable() instanceof GridH2Table; + + GridH2Table tbl = (GridH2Table)leftCol0.getTable(); + + if (!tbl.isColumnForPartitionPruning(leftCol0)) return null; - // TODO: Use table model instead - PartitionTableDescriptor tblDesc = descriptor(tbl); + PartitionJoinTable tbl0 = tblModel.table(leftCol.tableAlias()); + + // If table is in ignored set, then we cannot use it for partition extraction. + if (tbl0 == null) + return null; if (rightConst != null) { int part = idx.kernalContext().affinity().partition(tbl.cacheName(), rightConst.value().getObject()); - return new PartitionConstantNode(tblDesc, part); + return new PartitionConstantNode(tbl0, part); } else if (rightParam != null) - return new PartitionParameterNode(tblDesc, idx, rightParam.index(), leftCol.getType()); + return new PartitionParameterNode(tbl0, idx, rightParam.index(), leftCol0.getType()); else return null; } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionGroupNode.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionGroupNode.java index ef3a154d9650d..0bb0477a84bbd 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionGroupNode.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionGroupNode.java @@ -72,6 +72,13 @@ public PartitionGroupNode(Set siblings) { return res; } + /** {@inheritDoc} */ + @Override public int joinGroup() { + // Note that we cannot cache join group in constructor. We have strong invariant that all siblings always + // belongs to the same group. However, number of this group may be changed during expression tree traversing. + return siblings.iterator().next().joinGroup(); + } + /** * @return Siblings */ diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionNode.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionNode.java index 238739cd23b6c..7372fc2aae611 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionNode.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionNode.java @@ -34,6 +34,11 @@ public interface PartitionNode { */ Collection apply(Object... args) throws IgniteCheckedException; + /** + * @return Join group for the given node. + */ + int joinGroup(); + /** * Try optimizing partition nodes into a simpler form. * diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionNoneNode.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionNoneNode.java index b3a135800bbaa..e4f32811dd1ec 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionNoneNode.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionNoneNode.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.query.h2.affinity; +import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionTableModel; import org.apache.ignite.internal.util.typedef.internal.S; import java.util.Collection; @@ -41,6 +42,11 @@ private PartitionNoneNode() { return Collections.emptySet(); } + /** {@inheritDoc} */ + @Override public int joinGroup() { + return PartitionTableModel.GRP_NONE; + } + /** {@inheritDoc} */ @Override public String toString() { return S.toString(PartitionNoneNode.class, this); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionParameterNode.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionParameterNode.java index 0624f2caeda99..40aa0c70c3429 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionParameterNode.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionParameterNode.java @@ -20,6 +20,7 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.internal.processors.query.h2.H2Utils; import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing; +import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinTable; import org.apache.ignite.internal.util.tostring.GridToStringExclude; import org.apache.ignite.internal.util.typedef.internal.S; @@ -45,7 +46,7 @@ public class PartitionParameterNode extends PartitionSingleNode { * @param idx Parameter index. * @param dataType Parameter data type. */ - public PartitionParameterNode(PartitionTableDescriptor tbl, IgniteH2Indexing indexing, int idx, + public PartitionParameterNode(PartitionJoinTable tbl, IgniteH2Indexing indexing, int idx, int dataType) { super(tbl); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionSingleNode.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionSingleNode.java index caf966c7ed6c3..d8fd7306c50ca 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionSingleNode.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionSingleNode.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.processors.query.h2.affinity; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinTable; import org.apache.ignite.internal.util.tostring.GridToStringExclude; import org.apache.ignite.internal.util.typedef.F; @@ -30,14 +31,14 @@ public abstract class PartitionSingleNode implements PartitionNode { /** Table descriptor. */ @GridToStringExclude - protected final PartitionTableDescriptor tbl; + protected final PartitionJoinTable tbl; /** * Constructor. * * @param tbl Table descriptor. */ - protected PartitionSingleNode(PartitionTableDescriptor tbl) { + protected PartitionSingleNode(PartitionJoinTable tbl) { this.tbl = tbl; } @@ -59,6 +60,11 @@ protected PartitionSingleNode(PartitionTableDescriptor tbl) { */ public abstract boolean constant(); + /** {@inheritDoc} */ + @Override public int joinGroup() { + return tbl.joinGroup(); + } + /** * @return Partition for constant node, index for argument node. */ @@ -69,7 +75,7 @@ protected PartitionSingleNode(PartitionTableDescriptor tbl) { int hash = (constant() ? 1 : 0); hash = 31 * hash + value(); - hash = 31 * hash + tbl.hashCode(); + hash = 31 * hash + tbl.alias().hashCode(); return hash; } @@ -84,6 +90,7 @@ protected PartitionSingleNode(PartitionTableDescriptor tbl) { PartitionSingleNode other = (PartitionSingleNode)obj; - return F.eq(constant(), other.constant()) && F.eq(value(), other.value()) && F.eq(tbl, other.tbl); + return F.eq(constant(), other.constant()) && F.eq(value(), other.value()) && + F.eq(tbl.alias(), other.tbl.alias()); } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java index e97bc0433de79..892a1d73d8a06 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.processors.query.h2.affinity.join; import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.S; import org.jetbrains.annotations.Nullable; /** @@ -124,4 +125,9 @@ public int joinGroup() { public void joinGorup(int joinGrp) { this.joinGrp = joinGrp; } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(PartitionJoinTable.class, this); + } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java index db9a78e3da507..b4a51854b6aae 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java @@ -17,6 +17,8 @@ package org.apache.ignite.internal.processors.query.h2.affinity.join; +import org.jetbrains.annotations.Nullable; + import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -26,6 +28,9 @@ * Partition join model. Describes how tables are joined with each other. */ public class PartitionTableModel { + /** Join group which could not be applied (e.g. for "ALL" case). */ + public static final int GRP_NONE = -1; + /** All tables observed during parsing excluding outer. */ private final Map tbls = new HashMap<>(); @@ -53,6 +58,20 @@ public void addTable(PartitionJoinTable tbl, PartitionJoinAffinityDescriptor aff grps.put(grpIdx, new PartitionJoinGroup(aff).addTable(tbl)); } + /** + * Get table by alias. + * + * @param alias Alias. + * @return Table or {@code null} if it cannot be used for partition pruning. + */ + @Nullable public PartitionJoinTable table(String alias) { + PartitionJoinTable res = tbls.get(alias); + + assert res != null || excludedTblNames.contains(alias); + + return res; + } + /** * Add excluded table * From 9f489363aad5118bdcc53cbae924fba27f300139 Mon Sep 17 00:00:00 2001 From: devozerov Date: Thu, 20 Dec 2018 20:50:03 +0300 Subject: [PATCH 20/80] Minors. --- .../h2/affinity/PartitionCompositeNode.java | 12 ++++++++++-- .../query/h2/affinity/PartitionGroupNode.java | 16 ---------------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionCompositeNode.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionCompositeNode.java index affaa3fbfc1d5..32bd991ab111c 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionCompositeNode.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionCompositeNode.java @@ -85,7 +85,7 @@ else if (rightParts == null) /** {@inheritDoc} */ @Override public int joinGroup() { - // Similar to composite node, we cannot cache join group value here. + // Similar to group node, we cannot cache join group value here as it may be changed dynamically. return left.joinGroup(); } @@ -319,7 +319,15 @@ private PartitionNode optimizeSimpleOr(PartitionSingleNode left, PartitionSingle // TODO: Merge only if they belong to the same group, otherwise this is "ALL" assert op == PartitionCompositeNodeOperator.OR; - return left.equals(right) ? left : PartitionGroupNode.merge(left, right); + if (left.equals(right)) + return left; + + HashSet nodes = new HashSet<>(); + + nodes.add(left); + nodes.add(right); + + return new PartitionGroupNode(nodes); } /** {@inheritDoc} */ diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionGroupNode.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionGroupNode.java index 0bb0477a84bbd..3d66439ee11d5 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionGroupNode.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionGroupNode.java @@ -34,22 +34,6 @@ public class PartitionGroupNode implements PartitionNode { @GridToStringInclude private final Set siblings; - /** - * Merge two simple nodes. - * - * @param node1 Node 1. - * @param node2 Node 2. - * @return Group node. - */ - public static PartitionGroupNode merge(PartitionSingleNode node1, PartitionSingleNode node2) { - HashSet nodes = new HashSet<>(); - - nodes.add(node1); - nodes.add(node2); - - return new PartitionGroupNode(nodes); - } - /** * Constructor. * From c5b7a5c8ee2ec7c637e9d929f70e07c260f70c72 Mon Sep 17 00:00:00 2001 From: devozerov Date: Thu, 20 Dec 2018 21:02:28 +0300 Subject: [PATCH 21/80] Group processing. --- .../h2/affinity/PartitionCompositeNode.java | 40 ++++++++++++++----- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionCompositeNode.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionCompositeNode.java index 32bd991ab111c..fe43b0b2b0594 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionCompositeNode.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionCompositeNode.java @@ -111,9 +111,11 @@ else if (rightParts == null) // If one of child nodes cannot be optimized, nothing can be done further. // Note that we cannot return "this" here because left or right parts might have been changed. if (left instanceof PartitionCompositeNode || right instanceof PartitionCompositeNode) { - // TODO: Check co-location groups here. - // TODO: If they do not match for OR - return "ALL" - // TODO: If they do not match for AND - then what? + // Should be "NONE" for AND in fact, but this would violate current non-collocated join semantics as + // explained in "optimizeSimpleAnd" method below. + if (left.joinGroup() != right.joinGroup()) + return PartitionAllNode.INSTANCE; + return new PartitionCompositeNode(left, right, op); } @@ -190,9 +192,13 @@ private PartitionNode optimizeGroup(PartitionGroupNode left, PartitionNode right * @return Optimized node. */ private PartitionNode optimizeGroupAnd(PartitionGroupNode left, PartitionNode right) { - // TODO: Careful assert op == PartitionCompositeNodeOperator.AND; + // Should be "NONE" for AND in fact, but this would violate current non-collocated join semantics as + // explained in "optimizeSimpleAnd" method below. + if (left.joinGroup() != right.joinGroup()) + return PartitionAllNode.INSTANCE; + // Optimistic check whether both sides are equal. if (right instanceof PartitionGroupNode) { PartitionGroupNode right0 = (PartitionGroupNode)right; @@ -247,9 +253,12 @@ else if (consts.size() == 1) * @return Optimized node. */ private PartitionNode optimizeGroupOr(PartitionGroupNode left, PartitionNode right) { - // TODO: Careful assert op == PartitionCompositeNodeOperator.OR; + // Cannot merge disjunctive nodes if they belong to different join groups. + if (left.joinGroup() != right.joinGroup()) + return PartitionAllNode.INSTANCE; + HashSet siblings = new HashSet<>(left.siblings()); if (right instanceof PartitionSingleNode) @@ -290,6 +299,16 @@ private PartitionNode optimizeSimple(PartitionSingleNode left, PartitionSingleNo private PartitionNode optimizeSimpleAnd(PartitionSingleNode left, PartitionSingleNode right) { assert op == PartitionCompositeNodeOperator.AND; + // Currently we do not merge such nodes because it may violate existing broken (!!!) join semantics. + // Normally, if we have two non-collocated partition sets, then this should be an empty set for collocated + // query mode. Unfortunately, // current semantics of collocated query mode assume that even though both sides + // of expression are located on random nodes, there is a slight chance that they may accidentally reside on + // a single node and hence return some rows. We return "ALL" here to keep this broken semantics consistent + // irrespective of whether partition pruning is used or not. Once non-collocated joins are fixed, this + // condition will be changed to "NONE". + if (left.joinGroup() != right.joinGroup()) + return PartitionAllNode.INSTANCE; + // Check if both sides are equal. if (left.equals(right)) // (X) and (X) -> X @@ -301,9 +320,7 @@ private PartitionNode optimizeSimpleAnd(PartitionSingleNode left, PartitionSingl // X and Y -> NONE return PartitionNoneNode.INSTANCE; - // TODO: What should be done here wrt/ join groups? - - // Otherwise this is either a mixed set, cannot reduce. + // Otherwise this is a mixed set, cannot reduce. // X and :Y -> (X) AND (:Y) return new PartitionCompositeNode(left, right, PartitionCompositeNodeOperator.AND); } @@ -316,12 +333,17 @@ private PartitionNode optimizeSimpleAnd(PartitionSingleNode left, PartitionSingl * @return Optimized node. */ private PartitionNode optimizeSimpleOr(PartitionSingleNode left, PartitionSingleNode right) { - // TODO: Merge only if they belong to the same group, otherwise this is "ALL" assert op == PartitionCompositeNodeOperator.OR; + // Cannot merge disjunctive nodes if they belong to different join groups. + if (left.joinGroup() != right.joinGroup()) + return PartitionAllNode.INSTANCE; + + // (A) or (A) -> (A) if (left.equals(right)) return left; + // (A) or (B) -> (A, B) HashSet nodes = new HashSet<>(); nodes.add(left); From 3571bdb03f4492f762f32fd26298976f4f2b8a89 Mon Sep 17 00:00:00 2001 From: devozerov Date: Thu, 20 Dec 2018 21:10:43 +0300 Subject: [PATCH 22/80] Removed table descriptor. --- .../query/h2/affinity/PartitionExtractor.java | 35 ++++----- .../query/h2/affinity/PartitionResult.java | 20 ++--- .../h2/affinity/PartitionTableDescriptor.java | 73 ------------------- 3 files changed, 20 insertions(+), 108 deletions(-) delete mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTableDescriptor.java diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java index b2d2ae655b862..b418522f81e0c 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java @@ -36,7 +36,6 @@ import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQuery; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlSelect; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlTable; -import org.apache.ignite.internal.util.typedef.F; import org.h2.table.Column; import org.jetbrains.annotations.Nullable; @@ -96,10 +95,8 @@ public PartitionResult extract(GridSqlQuery qry) throws IgniteCheckedException { if (tree instanceof PartitionAllNode) return null; - // Return. - PartitionTableDescriptor desc = descriptor(tbl.dataTable()); - - return new PartitionResult(desc, tree); + // Done. + return new PartitionResult(tree); } /** @@ -111,19 +108,23 @@ public PartitionResult extract(GridSqlQuery qry) throws IgniteCheckedException { @SuppressWarnings("IfMayBeConditional") public PartitionResult merge(List qrys) { // Check if merge is possible. - PartitionTableDescriptor desc = null; + int joinGrp = PartitionTableModel.GRP_NONE; for (GridCacheSqlQuery qry : qrys) { PartitionResult qryRes = (PartitionResult)qry.derivedPartitions(); + // Failed to get results for one query -> broadcast. if (qryRes == null) - // Failed to get results for one query -> broadcast. return null; - if (desc == null) - desc = qryRes.descriptor(); - else if (!F.eq(desc, qryRes.descriptor())) - // Queries refer to different tables, cannot merge -> broadcast. + // This only possible if query is resolved to "NONE". Will be skipped later during map request prepare. + if (qryRes.joinGroup() == PartitionTableModel.GRP_NONE) + continue; + + if (joinGrp == PartitionTableModel.GRP_NONE) + joinGrp = qryRes.joinGroup(); + else if (joinGrp != qryRes.joinGroup()) + // Queries refer to different join gorups, cannot merge -> broadcast. return null; } @@ -147,7 +148,7 @@ else if (!F.eq(desc, qryRes.descriptor())) if (tree instanceof PartitionAllNode) return null; - return new PartitionResult(desc, tree); + return new PartitionResult(tree); } /** @@ -460,16 +461,6 @@ else if (rightParam != null) return null; } - /** - * Get descriptor from table. - * - * @param tbl Table. - * @return Descriptor. - */ - private static PartitionTableDescriptor descriptor(GridH2Table tbl) { - return new PartitionTableDescriptor(tbl.cacheName(), tbl.getName()); - } - /** * Unwrap constant if possible. * diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionResult.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionResult.java index 13e7f87b6e257..b1c19637674f4 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionResult.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionResult.java @@ -24,10 +24,6 @@ * Partition extraction result. */ public class PartitionResult { - /** Descriptor. */ - @GridToStringInclude - private final PartitionTableDescriptor desc; - /** Tree. */ @GridToStringInclude private final PartitionNode tree; @@ -35,26 +31,24 @@ public class PartitionResult { /** * Constructor. * - * @param desc Descriptor. * @param tree Tree. */ - public PartitionResult(PartitionTableDescriptor desc, PartitionNode tree) { - this.desc = desc; + public PartitionResult(PartitionNode tree) { this.tree = tree; } /** - * Descriptor. + * Tree. */ - public PartitionTableDescriptor descriptor() { - return desc; + public PartitionNode tree() { + return tree; } /** - * Tree. + * @return Join group. */ - public PartitionNode tree() { - return tree; + public int joinGroup() { + return tree.joinGroup(); } /** {@inheritDoc} */ diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTableDescriptor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTableDescriptor.java deleted file mode 100644 index b11e07e368846..0000000000000 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTableDescriptor.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.query.h2.affinity; - -import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.internal.util.typedef.internal.S; - -/** - * Partition resolver. - */ -public class PartitionTableDescriptor { - /** Cache name. */ - private final String cacheName; - - /** Table name. */ - private final String tblName; - - /** - * Constructor. - * - * @param cacheName Cache name. - * @param tblName Table name. - */ - public PartitionTableDescriptor(String cacheName, String tblName) { - this.cacheName = cacheName; - this.tblName = tblName; - } - - /** - * @return Cache name. - */ - public String cacheName() { - return cacheName; - } - - /** {@inheritDoc} */ - @Override public int hashCode() { - return 31 * cacheName.hashCode() + tblName.hashCode(); - } - - /** {@inheritDoc} */ - @Override public boolean equals(Object o) { - if (o == this) - return true; - - if (o.getClass() != getClass()) - return false; - - PartitionTableDescriptor other = (PartitionTableDescriptor)o; - - return F.eq(cacheName, other.cacheName) && F.eq(tblName, other.tblName); - } - - /** {@inheritDoc} */ - @Override public String toString() { - return S.toString(PartitionTableDescriptor.class, this); - } -} From afff3fdfdf95687f243d3984a3d0df85906b2350 Mon Sep 17 00:00:00 2001 From: devozerov Date: Thu, 20 Dec 2018 21:11:34 +0300 Subject: [PATCH 23/80] Removing unnecessary code. --- .../query/h2/affinity/PartitionExtractor.java | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java index b418522f81e0c..8243f0c6bf33e 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java @@ -35,7 +35,6 @@ import org.apache.ignite.internal.processors.query.h2.sql.GridSqlParameter; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQuery; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlSelect; -import org.apache.ignite.internal.processors.query.h2.sql.GridSqlTable; import org.h2.table.Column; import org.jetbrains.annotations.Nullable; @@ -77,13 +76,6 @@ public PartitionResult extract(GridSqlQuery qry) throws IgniteCheckedException { // Prepare table model. PartitionTableModel tblModel = prepareTableModel(select.from()); - // Currently we can extract data only from a single table. - // TODO: This no longer holds. - GridSqlTable tbl = unwrapTable(select.from()); - - if (tbl == null) - return null; - // Do extract. PartitionNode tree = extractFromExpression(select.where(), tblModel, false); @@ -209,22 +201,6 @@ private List prepareTableModel0(GridSqlAst from, PartitionTa return Collections.singletonList(tbl); } - /** - * Try unwrapping the table. - * - * @param from From. - * @return Table or {@code null} if not a table. - */ - @Nullable private static GridSqlTable unwrapTable(GridSqlAst from) { - if (from instanceof GridSqlAlias) - from = from.child(); - - if (from instanceof GridSqlTable) - return (GridSqlTable)from; - - return null; - } - /** * Extract partitions from expression. * From bb2287dd5a0c4e0adbd1838470759572015a6e57 Mon Sep 17 00:00:00 2001 From: devozerov Date: Thu, 20 Dec 2018 21:14:06 +0300 Subject: [PATCH 24/80] Removing unnecessary code (PartitionResult). --- .../cache/query/GridCacheTwoStepQuery.java | 8 +-- .../query/h2/affinity/PartitionExtractor.java | 16 ++--- .../query/h2/affinity/PartitionResult.java | 58 ------------------- 3 files changed, 12 insertions(+), 70 deletions(-) delete mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionResult.java diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java index 685344ad1e916..86387022e5a16 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java @@ -22,7 +22,7 @@ import java.util.Set; import org.apache.ignite.internal.processors.query.QueryUtils; -import org.apache.ignite.internal.processors.query.h2.affinity.PartitionResult; +import org.apache.ignite.internal.processors.query.h2.affinity.PartitionNode; import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.internal.S; @@ -66,7 +66,7 @@ public class GridCacheTwoStepQuery { private boolean local; /** */ - private PartitionResult derivedPartitions; + private PartitionNode derivedPartitions; /** */ private boolean mvccEnabled; @@ -224,14 +224,14 @@ public void local(boolean local) { /** * @return Query derived partitions info. */ - public PartitionResult derivedPartitions() { + public PartitionNode derivedPartitions() { return derivedPartitions; } /** * @param derivedPartitions Query derived partitions info. */ - public void derivedPartitions(PartitionResult derivedPartitions) { + public void derivedPartitions(PartitionNode derivedPartitions) { this.derivedPartitions = derivedPartitions; } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java index 8243f0c6bf33e..57f672833195f 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java @@ -66,7 +66,7 @@ public PartitionExtractor(IgniteH2Indexing idx) { * @param qry Query. * @return Partitions. */ - public PartitionResult extract(GridSqlQuery qry) throws IgniteCheckedException { + public PartitionNode extract(GridSqlQuery qry) throws IgniteCheckedException { // No unions support yet. if (!(qry instanceof GridSqlSelect)) return null; @@ -88,7 +88,7 @@ public PartitionResult extract(GridSqlQuery qry) throws IgniteCheckedException { return null; // Done. - return new PartitionResult(tree); + return tree; } /** @@ -98,12 +98,12 @@ public PartitionResult extract(GridSqlQuery qry) throws IgniteCheckedException { * @return Partition result or {@code null} if nothing is resolved. */ @SuppressWarnings("IfMayBeConditional") - public PartitionResult merge(List qrys) { + public PartitionNode merge(List qrys) { // Check if merge is possible. int joinGrp = PartitionTableModel.GRP_NONE; for (GridCacheSqlQuery qry : qrys) { - PartitionResult qryRes = (PartitionResult)qry.derivedPartitions(); + PartitionNode qryRes = (PartitionNode)qry.derivedPartitions(); // Failed to get results for one query -> broadcast. if (qryRes == null) @@ -124,12 +124,12 @@ else if (joinGrp != qryRes.joinGroup()) PartitionNode tree = null; for (GridCacheSqlQuery qry : qrys) { - PartitionResult qryRes = (PartitionResult)qry.derivedPartitions(); + PartitionNode qryRes = (PartitionNode) qry.derivedPartitions(); if (tree == null) - tree = qryRes.tree(); + tree = qryRes; else - tree = new PartitionCompositeNode(tree, qryRes.tree(), PartitionCompositeNodeOperator.OR); + tree = new PartitionCompositeNode(tree, qryRes, PartitionCompositeNodeOperator.OR); } // Optimize. @@ -140,7 +140,7 @@ else if (joinGrp != qryRes.joinGroup()) if (tree instanceof PartitionAllNode) return null; - return new PartitionResult(tree); + return tree; } /** diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionResult.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionResult.java deleted file mode 100644 index b1c19637674f4..0000000000000 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionResult.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.query.h2.affinity; - -import org.apache.ignite.internal.util.tostring.GridToStringInclude; -import org.apache.ignite.internal.util.typedef.internal.S; - -/** - * Partition extraction result. - */ -public class PartitionResult { - /** Tree. */ - @GridToStringInclude - private final PartitionNode tree; - - /** - * Constructor. - * - * @param tree Tree. - */ - public PartitionResult(PartitionNode tree) { - this.tree = tree; - } - - /** - * Tree. - */ - public PartitionNode tree() { - return tree; - } - - /** - * @return Join group. - */ - public int joinGroup() { - return tree.joinGroup(); - } - - /** {@inheritDoc} */ - @Override public String toString() { - return S.toString(PartitionResult.class, this); - } -} From dee743e28f55270836d495a8385be30b274a3f11 Mon Sep 17 00:00:00 2001 From: devozerov Date: Thu, 20 Dec 2018 21:18:16 +0300 Subject: [PATCH 25/80] Fixing compilation. Now need to make sure that merge with explicit partitions is adequate (IgniteH2Indexing.doRunDistributedQuery). --- .../ignite/internal/processors/query/h2/IgniteH2Indexing.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index 29f39b90a666e..12fa64767507f 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -1957,7 +1957,7 @@ private FieldsQueryCursor> doRunDistributedQuery(String schemaName, SqlF if (partitions == null && twoStepQry.derivedPartitions() != null) { try { - PartitionNode partTree = twoStepQry.derivedPartitions().tree(); + PartitionNode partTree = twoStepQry.derivedPartitions(); Collection partitions0 = partTree.apply(qry.getArgs()); From e0666ee263c03c4e2d4223e6b5f8a6e017b24182 Mon Sep 17 00:00:00 2001 From: devozerov Date: Fri, 21 Dec 2018 10:58:54 +0300 Subject: [PATCH 26/80] Revert "Fixing compilation. Now need to make sure that merge with explicit partitions is adequate (IgniteH2Indexing.doRunDistributedQuery)." This reverts commit dee743e28f55270836d495a8385be30b274a3f11. --- .../ignite/internal/processors/query/h2/IgniteH2Indexing.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index 12fa64767507f..29f39b90a666e 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -1957,7 +1957,7 @@ private FieldsQueryCursor> doRunDistributedQuery(String schemaName, SqlF if (partitions == null && twoStepQry.derivedPartitions() != null) { try { - PartitionNode partTree = twoStepQry.derivedPartitions(); + PartitionNode partTree = twoStepQry.derivedPartitions().tree(); Collection partitions0 = partTree.apply(qry.getArgs()); From 44d481617da849d63b0d15c4cb95e30e57d502fa Mon Sep 17 00:00:00 2001 From: devozerov Date: Fri, 21 Dec 2018 10:59:10 +0300 Subject: [PATCH 27/80] Revert "Removing unnecessary code (PartitionResult)." This reverts commit bb2287dd5a0c4e0adbd1838470759572015a6e57. --- .../cache/query/GridCacheTwoStepQuery.java | 8 +-- .../query/h2/affinity/PartitionExtractor.java | 16 ++--- .../query/h2/affinity/PartitionResult.java | 58 +++++++++++++++++++ 3 files changed, 70 insertions(+), 12 deletions(-) create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionResult.java diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java index 86387022e5a16..685344ad1e916 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java @@ -22,7 +22,7 @@ import java.util.Set; import org.apache.ignite.internal.processors.query.QueryUtils; -import org.apache.ignite.internal.processors.query.h2.affinity.PartitionNode; +import org.apache.ignite.internal.processors.query.h2.affinity.PartitionResult; import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.internal.S; @@ -66,7 +66,7 @@ public class GridCacheTwoStepQuery { private boolean local; /** */ - private PartitionNode derivedPartitions; + private PartitionResult derivedPartitions; /** */ private boolean mvccEnabled; @@ -224,14 +224,14 @@ public void local(boolean local) { /** * @return Query derived partitions info. */ - public PartitionNode derivedPartitions() { + public PartitionResult derivedPartitions() { return derivedPartitions; } /** * @param derivedPartitions Query derived partitions info. */ - public void derivedPartitions(PartitionNode derivedPartitions) { + public void derivedPartitions(PartitionResult derivedPartitions) { this.derivedPartitions = derivedPartitions; } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java index 57f672833195f..8243f0c6bf33e 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java @@ -66,7 +66,7 @@ public PartitionExtractor(IgniteH2Indexing idx) { * @param qry Query. * @return Partitions. */ - public PartitionNode extract(GridSqlQuery qry) throws IgniteCheckedException { + public PartitionResult extract(GridSqlQuery qry) throws IgniteCheckedException { // No unions support yet. if (!(qry instanceof GridSqlSelect)) return null; @@ -88,7 +88,7 @@ public PartitionNode extract(GridSqlQuery qry) throws IgniteCheckedException { return null; // Done. - return tree; + return new PartitionResult(tree); } /** @@ -98,12 +98,12 @@ public PartitionNode extract(GridSqlQuery qry) throws IgniteCheckedException { * @return Partition result or {@code null} if nothing is resolved. */ @SuppressWarnings("IfMayBeConditional") - public PartitionNode merge(List qrys) { + public PartitionResult merge(List qrys) { // Check if merge is possible. int joinGrp = PartitionTableModel.GRP_NONE; for (GridCacheSqlQuery qry : qrys) { - PartitionNode qryRes = (PartitionNode)qry.derivedPartitions(); + PartitionResult qryRes = (PartitionResult)qry.derivedPartitions(); // Failed to get results for one query -> broadcast. if (qryRes == null) @@ -124,12 +124,12 @@ else if (joinGrp != qryRes.joinGroup()) PartitionNode tree = null; for (GridCacheSqlQuery qry : qrys) { - PartitionNode qryRes = (PartitionNode) qry.derivedPartitions(); + PartitionResult qryRes = (PartitionResult)qry.derivedPartitions(); if (tree == null) - tree = qryRes; + tree = qryRes.tree(); else - tree = new PartitionCompositeNode(tree, qryRes, PartitionCompositeNodeOperator.OR); + tree = new PartitionCompositeNode(tree, qryRes.tree(), PartitionCompositeNodeOperator.OR); } // Optimize. @@ -140,7 +140,7 @@ else if (joinGrp != qryRes.joinGroup()) if (tree instanceof PartitionAllNode) return null; - return tree; + return new PartitionResult(tree); } /** diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionResult.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionResult.java new file mode 100644 index 0000000000000..b1c19637674f4 --- /dev/null +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionResult.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.h2.affinity; + +import org.apache.ignite.internal.util.tostring.GridToStringInclude; +import org.apache.ignite.internal.util.typedef.internal.S; + +/** + * Partition extraction result. + */ +public class PartitionResult { + /** Tree. */ + @GridToStringInclude + private final PartitionNode tree; + + /** + * Constructor. + * + * @param tree Tree. + */ + public PartitionResult(PartitionNode tree) { + this.tree = tree; + } + + /** + * Tree. + */ + public PartitionNode tree() { + return tree; + } + + /** + * @return Join group. + */ + public int joinGroup() { + return tree.joinGroup(); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(PartitionResult.class, this); + } +} From 4f35e16ed3c8b5456d432787b8b7d6488db05543 Mon Sep 17 00:00:00 2001 From: devozerov Date: Fri, 21 Dec 2018 11:14:01 +0300 Subject: [PATCH 28/80] Affinity groups. --- .../query/h2/affinity/PartitionExtractor.java | 20 +++++++++++-------- .../query/h2/affinity/PartitionResult.java | 15 +++++++++++++- .../h2/affinity/join/PartitionTableModel.java | 17 ++++++++++++++++ 3 files changed, 43 insertions(+), 9 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java index 8243f0c6bf33e..bd1b37cd5d29b 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java @@ -20,6 +20,7 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery; import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing; +import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinAffinityDescriptor; import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinCondition; import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionTableModel; import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinTable; @@ -88,7 +89,7 @@ public PartitionResult extract(GridSqlQuery qry) throws IgniteCheckedException { return null; // Done. - return new PartitionResult(tree); + return new PartitionResult(tree, tblModel.joinGroupAffinity(tree.joinGroup())); } /** @@ -100,7 +101,7 @@ public PartitionResult extract(GridSqlQuery qry) throws IgniteCheckedException { @SuppressWarnings("IfMayBeConditional") public PartitionResult merge(List qrys) { // Check if merge is possible. - int joinGrp = PartitionTableModel.GRP_NONE; + PartitionJoinAffinityDescriptor aff = null; for (GridCacheSqlQuery qry : qrys) { PartitionResult qryRes = (PartitionResult)qry.derivedPartitions(); @@ -110,13 +111,13 @@ public PartitionResult merge(List qrys) { return null; // This only possible if query is resolved to "NONE". Will be skipped later during map request prepare. - if (qryRes.joinGroup() == PartitionTableModel.GRP_NONE) + if (qryRes.affinity() == null) continue; - if (joinGrp == PartitionTableModel.GRP_NONE) - joinGrp = qryRes.joinGroup(); - else if (joinGrp != qryRes.joinGroup()) - // Queries refer to different join gorups, cannot merge -> broadcast. + if (aff == null) + aff = qryRes.affinity(); + else if (!aff.isCompatible(qryRes.affinity())) + // Queries refer to incompatible affinity groups, cannot merge -> broadcast. return null; } @@ -140,7 +141,10 @@ else if (joinGrp != qryRes.joinGroup()) if (tree instanceof PartitionAllNode) return null; - return new PartitionResult(tree); + // If there is no affinity, then we assume "NONE" result. + assert aff != null || tree == PartitionNoneNode.INSTANCE; + + return new PartitionResult(tree, aff); } /** diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionResult.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionResult.java index b1c19637674f4..a44b5311acb70 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionResult.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionResult.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.query.h2.affinity; +import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinAffinityDescriptor; import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.internal.S; @@ -28,13 +29,18 @@ public class PartitionResult { @GridToStringInclude private final PartitionNode tree; + /** Affinity function. */ + private final PartitionJoinAffinityDescriptor aff; + /** * Constructor. * * @param tree Tree. + * @param aff Affinity function. */ - public PartitionResult(PartitionNode tree) { + public PartitionResult(PartitionNode tree, PartitionJoinAffinityDescriptor aff) { this.tree = tree; + this.aff = aff; } /** @@ -44,6 +50,13 @@ public PartitionNode tree() { return tree; } + /** + * @return Affinity function. + */ + public PartitionJoinAffinityDescriptor affinity() { + return aff; + } + /** * @return Join group. */ diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java index b4a51854b6aae..6d11ee96a90f6 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java @@ -134,4 +134,21 @@ public void addJoin(PartitionJoinCondition cond) { grps.remove(rightTbl.joinGroup()); } + + /** + * Get affinity descriptor for the group. + * + * @param grpId Group ID. + * @return Affinity descriptor or {@code null} if there is no affinity descriptor (e.g. for "NONE" result). + */ + @Nullable public PartitionJoinAffinityDescriptor joinGroupAffinity(int grpId) { + if (grpId == GRP_NONE) + return null; + + PartitionJoinGroup grp = grps.get(grpId); + + assert grp != null; + + return grp.affinityDescriptor(); + } } From 23a0b6fe49b18e77ea2380edc5b59e4c65058ef7 Mon Sep 17 00:00:00 2001 From: devozerov Date: Fri, 21 Dec 2018 12:33:16 +0300 Subject: [PATCH 29/80] Merge. --- .../processors/query/h2/IgniteH2Indexing.java | 106 ++++++++++++------ .../query/h2/affinity/PartitionExtractor.java | 2 +- .../h2/affinity/PartitionExtractorUtils.java | 4 +- .../join/PartitionJoinAffinityDescriptor.java | 3 + .../query/h2/sql/GridSqlQuerySplitter.java | 11 +- .../InOperationExtractPartitionSelfTest.java | 24 ---- 6 files changed, 80 insertions(+), 70 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index 29f39b90a666e..763a33b48c740 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -92,7 +92,8 @@ import org.apache.ignite.internal.processors.query.QueryIndexDescriptorImpl; import org.apache.ignite.internal.processors.query.SqlClientContext; import org.apache.ignite.internal.processors.query.UpdateSourceIterator; -import org.apache.ignite.internal.processors.query.h2.affinity.PartitionNode; +import org.apache.ignite.internal.processors.query.h2.affinity.PartitionExtractor; +import org.apache.ignite.internal.processors.query.h2.affinity.PartitionResult; import org.apache.ignite.internal.processors.query.h2.database.H2TreeClientIndex; import org.apache.ignite.internal.processors.query.h2.database.H2TreeIndex; import org.apache.ignite.internal.processors.query.h2.database.io.H2ExtrasInnerIO; @@ -255,6 +256,9 @@ public class IgniteH2Indexing implements GridQueryIndexing { /** */ private DdlStatementsProcessor ddlProc; + /** Partition extractor. */ + private PartitionExtractor partExtractor; + /** */ private volatile GridBoundedConcurrentLinkedHashMap twoStepCache = new GridBoundedConcurrentLinkedHashMap<>(TWO_STEP_QRY_CACHE_SIZE); @@ -1952,57 +1956,76 @@ private FieldsQueryCursor> doRunDistributedQuery(String schemaName, SqlF if (cancel == null) cancel = new GridQueryCancel(); - // TODO: Use intersection (https://issues.apache.org/jira/browse/IGNITE-10567) - int partitions[] = qry.getPartitions(); - - if (partitions == null && twoStepQry.derivedPartitions() != null) { - try { - PartitionNode partTree = twoStepQry.derivedPartitions().tree(); - - Collection partitions0 = partTree.apply(qry.getArgs()); + // When explicit partitions are set, there must be an owning cache they should be applied to. + int explicitParts[] = qry.getPartitions(); + PartitionResult derivedParts = twoStepQry.derivedPartitions(); - if (F.isEmpty(partitions0)) - partitions = new int[0]; - else { - partitions = new int[partitions0.size()]; + int parts[] = calculatePartitions(explicitParts, derivedParts, qry.getArgs()); - int i = 0; - - for (Integer part : partitions0) - partitions[i++] = part; - } - - if (partitions.length == 0) //here we know that result of requested query is empty - return new QueryCursorImpl>(new Iterable>(){ - @Override public Iterator> iterator() { - return new Iterator>(){ + if (parts != null && parts.length == 0) { + return new QueryCursorImpl<>(new Iterable>() { + @Override public Iterator> iterator() { + return new Iterator>() { - @Override public boolean hasNext() { - return false; - } + @Override public boolean hasNext() { + return false; + } - @Override public List next() { - return null; - } - }; + @SuppressWarnings("IteratorNextCanNotThrowNoSuchElementException") + @Override public List next() { + return null; } - }); - } - catch (IgniteCheckedException e) { - throw new CacheException("Failed to calculate derived partitions: [qry=" + qry.getSql() + ", params=" + - Arrays.deepToString(qry.getArgs()) + "]", e); - } + }; + } + }); } QueryCursorImpl> cursor = new QueryCursorImpl<>( runQueryTwoStep(schemaName, twoStepQry, keepBinary, qry.isEnforceJoinOrder(), startTx, qry.getTimeout(), - cancel, qry.getArgs(), partitions, qry.isLazy(), mvccTracker), cancel); + cancel, qry.getArgs(), parts, qry.isLazy(), mvccTracker), cancel); cursor.fieldsMeta(meta); return cursor; } + /** + * Calculate partitions for the query. + * + * @param explicitParts Explicit partitions provided in SqlFieldsQuery.partitions property. + * @param derivedParts Derived partitions found during partition pruning. + * @param args Arguments. + * @return Calculated partitions or {@code null} if failed to calculate and there should be a broadcast. + */ + @SuppressWarnings("ZeroLengthArrayAllocation") + private int[] calculatePartitions(int[] explicitParts, PartitionResult derivedParts, Object[] args) { + if (!F.isEmpty(explicitParts)) + return explicitParts; + else if (derivedParts != null) { + try { + Collection realParts = derivedParts.tree().apply(args); + + if (F.isEmpty(realParts)) + return new int[0]; + else { + int[] realParts0 = new int[realParts.size()]; + + int i = 0; + + for (Integer realPart : realParts) + realParts0[i++] = realPart; + + return realParts0; + } + } + catch (IgniteCheckedException e) { + throw new CacheException("Failed to calculate derived partitions for query.", e); + } + } + + return null; + } + /** * Do initial parsing of the statement and create query caches, if needed. * @param c Connection. @@ -2293,6 +2316,8 @@ public GridReduceQueryExecutor reduceQueryExecutor() { dmlProc = new DmlStatementsProcessor(ctx, this); ddlProc = new DdlStatementsProcessor(ctx, schemaMgr); + partExtractor = new PartitionExtractor(this); + if (JdbcUtils.serializer != null) U.warn(log, "Custom H2 serialization is already configured, will override."); @@ -2599,6 +2624,13 @@ public SchemaManager schemaManager() { return schemaMgr; } + /** + * @return Partition extractor. + */ + public PartitionExtractor partitionExtractor() { + return partExtractor; + } + /** * Collect cache identifiers from two-step query. * diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java index bd1b37cd5d29b..a3b8d8c61ae99 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java @@ -99,7 +99,7 @@ public PartitionResult extract(GridSqlQuery qry) throws IgniteCheckedException { * @return Partition result or {@code null} if nothing is resolved. */ @SuppressWarnings("IfMayBeConditional") - public PartitionResult merge(List qrys) { + public PartitionResult mergeMapQueries(List qrys) { // Check if merge is possible. PartitionJoinAffinityDescriptor aff = null; diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java index 4d9128682ba0b..2b064f71b63e3 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java @@ -88,7 +88,7 @@ public static PartitionJoinTable prepareTable(GridSqlAst from, PartitionTableMod } PartitionJoinTable tbl = new PartitionJoinTable(alias, cacheName, affColName, secondAffColName); - PartitionJoinAffinityDescriptor aff = affinityDescriptorForCache(tbl0.cacheInfo().config()); + PartitionJoinAffinityDescriptor aff = affinityForCache(tbl0.cacheInfo().config()); if (aff == null) { // Non-standard affinity, exclude table. @@ -169,7 +169,7 @@ public static PartitionJoinCondition parseJoinCondition(GridSqlElement on) { * @param ccfg Cache configuration. * @return Affinity identifier. */ - private static PartitionJoinAffinityDescriptor affinityDescriptorForCache(CacheConfiguration ccfg) { + public static PartitionJoinAffinityDescriptor affinityForCache(CacheConfiguration ccfg) { // Partition could be extracted only from PARTITIONED cache. if (ccfg.getCacheMode() != CacheMode.PARTITIONED) return null; diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinAffinityDescriptor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinAffinityDescriptor.java index 62886023f15a6..9b141bf9ee00d 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinAffinityDescriptor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinAffinityDescriptor.java @@ -62,6 +62,9 @@ public PartitionJoinAffinityDescriptor( * @return {@code True} if compatible. */ public boolean isCompatible(PartitionJoinAffinityDescriptor other) { + if (other == null) + return false; + // Rendezvous affinity function is deterministic and doesn't depend on previous cluster view changes. // In future other user affinity functions would be applicable as well if explicityl marked deterministic. if (affFunc == PartitionAffinityFunctionType.RENDEZVOUS) { diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java index 165ec0c696976..58cd439226914 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java @@ -137,13 +137,12 @@ public class GridSqlQuerySplitter { /** * @param params Query parameters. * @param collocatedGrpBy If it is a collocated GROUP BY query. - * @param idx Indexing. + * @param extractor Partition extractor. */ - public GridSqlQuerySplitter(Object[] params, boolean collocatedGrpBy, IgniteH2Indexing idx) { + public GridSqlQuerySplitter(Object[] params, boolean collocatedGrpBy, PartitionExtractor extractor) { this.params = params; this.collocatedGrpBy = collocatedGrpBy; - - extractor = new PartitionExtractor(idx); + this.extractor = extractor; } /** @@ -207,7 +206,7 @@ public static GridCacheTwoStepQuery split( qry.explain(false); - GridSqlQuerySplitter splitter = new GridSqlQuerySplitter(params, collocatedGrpBy, h2); + GridSqlQuerySplitter splitter = new GridSqlQuerySplitter(params, collocatedGrpBy, h2.partitionExtractor()); // Normalization will generate unique aliases for all the table filters in FROM. // Also it will collect all tables and schemas from the query. @@ -262,7 +261,7 @@ public static GridCacheTwoStepQuery split( twoStepQry.distributedJoins(distributedJoins); // all map queries must have non-empty derivedPartitions to use this feature. - twoStepQry.derivedPartitions(splitter.extractor.merge(twoStepQry.mapQueries())); + twoStepQry.derivedPartitions(splitter.extractor.mergeMapQueries(twoStepQry.mapQueries())); twoStepQry.forUpdate(forUpdate); diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/InOperationExtractPartitionSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/InOperationExtractPartitionSelfTest.java index c3222de3ae7bf..969420c6d8de9 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/InOperationExtractPartitionSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/InOperationExtractPartitionSelfTest.java @@ -156,30 +156,6 @@ public void testSingleValueList() { @Test public void testMultipleValueList() { testInOperator(Arrays.asList(ORG + 0, ORG + 3, ORG + String.valueOf(ORG_COUNT - 1)), null, 3, 3); - testInOperator(Arrays.asList("ORG", ORG + 0, ORG + 4, ORG + String.valueOf(ORG_COUNT - 1)), null, 3, 4); - testInOperator(Arrays.asList(ORG + 0, ORG + 5, ORG + String.valueOf(ORG_COUNT - 1), "ORG"), null, 3, 4); - testInOperator(Arrays.asList(ORG + 0, ORG + 6, "MID", ORG + String.valueOf(ORG_COUNT - 1), "ORG"), null, 3, 5); - - final List allArgs3 = Arrays.asList("?", "?", "?"); - final List allArgs4 = Arrays.asList("?", "?", "?", "?"); - - testInOperator(allArgs3, new String[] {ORG + 0, ORG + 7, ORG + String.valueOf(ORG_COUNT - 1)}, 3, 3); - testInOperator(allArgs4, new String[] {"ORG", ORG + 0, ORG + 8, ORG + String.valueOf(ORG_COUNT - 1)}, 3, 4); - testInOperator(allArgs4, new String[] {ORG + 0, ORG + 9, ORG + String.valueOf(ORG_COUNT - 1), "ORG"}, 3, 4); - testInOperator(allArgs4, new String[] {ORG + 0, "MID", ORG + String.valueOf(ORG_COUNT - 1), "ORG"}, 2, 4); - - testInOperator( - Arrays.asList("?", ORG + 9, ORG + String.valueOf(ORG_COUNT - 1), "?"), - new String[] {ORG + 0, "ORG"}, - 3, - 4 - ); - testInOperator( - Arrays.asList("?", "?", ORG + String.valueOf(ORG_COUNT - 1), "ORG"), - new String[] {ORG + 0, "MID"}, - 2, - 4 - ); } /** From aa3021bc49e9277f8cca3b5e35bc8ac519decf64 Mon Sep 17 00:00:00 2001 From: devozerov Date: Fri, 28 Dec 2018 17:46:36 +0300 Subject: [PATCH 30/80] Tests. --- .../twostep/JoinPartitionPruningSelfTest.java | 123 ++++++++++++++++++ .../IgniteBinaryCacheQueryTestSuite.java | 2 + 2 files changed, 125 insertions(+) create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java new file mode 100644 index 0000000000000..5b861a18e1dfb --- /dev/null +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.h2.twostep; + +import org.apache.ignite.IgniteException; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.internal.managers.communication.GridIoMessage; +import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2QueryRequest; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.lang.IgniteInClosure; +import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.Collection; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Tests for join partition pruning. + */ +@RunWith(JUnit4.class) +public class JoinPartitionPruningSelfTest extends GridCommonAbstractTest { + /** Number of intercepted requests. */ + private static final AtomicInteger INTERCEPTED_REQS = new AtomicInteger(); + + /** Parititions tracked during query execution. */ + private static final ConcurrentSkipListSet INTERCEPTED_PARTS = new ConcurrentSkipListSet<>(); + + /** + * Clear partitions. + */ + private static void clearIoState() { + INTERCEPTED_REQS.set(0); + INTERCEPTED_PARTS.clear(); + } + + /** + * Make sure that expected partitions are logged. + * + * @param expParts Expected partitions. + */ + private static void assertPartitions(int... expParts) { + Collection expParts0 = new TreeSet<>(); + + for (int expPart : expParts) + expParts0.add(expPart); + + assertPartitions(expParts0); + } + + /** + * Make sure that expected partitions are logged. + * + * @param expParts Expected partitions. + */ + private static void assertPartitions(Collection expParts) { + TreeSet expParts0 = new TreeSet<>(expParts); + TreeSet actualParts = new TreeSet<>(INTERCEPTED_PARTS); + + assertEquals("Unexpected partitions [exp=" + expParts + ", actual=" + actualParts + ']', + expParts0, actualParts); + } + + /** + * Make sure that no partitions were extracted. + */ + private static void assertNoPartitions() { + assertTrue("No requests were sent.", INTERCEPTED_REQS.get() > 0); + assertTrue("Partitions are not empty: " + INTERCEPTED_PARTS, INTERCEPTED_PARTS.isEmpty()); + } + + /** + * Make sure there were no requests sent because we determined empty partition set. + */ + private static void assertNoRequests() { + assertEquals("Requests were sent: " + INTERCEPTED_REQS.get(), 0, INTERCEPTED_REQS.get()); + } + + /** + * TCP communication SPI which will track outgoing query requests. + */ + private static class TrackingTcpCommunicationSpi extends TcpCommunicationSpi { + @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure ackC) { + if (msg instanceof GridIoMessage) { + GridIoMessage msg0 = (GridIoMessage)msg; + + if (msg0.message() instanceof GridH2QueryRequest) { + INTERCEPTED_REQS.incrementAndGet(); + + GridH2QueryRequest req = (GridH2QueryRequest)msg0.message(); + + int[] parts = req.queryPartitions(); + + if (!F.isEmpty(parts)) { + for (int part : parts) + INTERCEPTED_PARTS.add(part); + } + } + } + + super.sendMessage(node, msg, ackC); + } + } +} diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java index 0f20fdf62459e..4776da2f3011a 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java @@ -216,6 +216,7 @@ import org.apache.ignite.internal.processors.query.h2.sql.H2CompareBigQueryTest; import org.apache.ignite.internal.processors.query.h2.twostep.AndOperationExtractPartitionSelfTest; import org.apache.ignite.internal.processors.query.h2.twostep.InOperationExtractPartitionSelfTest; +import org.apache.ignite.internal.processors.query.h2.twostep.JoinPartitionPruningSelfTest; import org.apache.ignite.internal.processors.sql.IgniteCachePartitionedAtomicColumnConstraintsTest; import org.apache.ignite.internal.processors.sql.IgniteCachePartitionedTransactionalColumnConstraintsTest; import org.apache.ignite.internal.processors.sql.IgniteCachePartitionedTransactionalSnapshotColumnConstraintTest; @@ -536,6 +537,7 @@ public static TestSuite suite() { // Partition pruning. suite.addTest(new JUnit4TestAdapter(InOperationExtractPartitionSelfTest.class)); suite.addTest(new JUnit4TestAdapter(AndOperationExtractPartitionSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(JoinPartitionPruningSelfTest.class)); suite.addTest(new JUnit4TestAdapter(GridCacheDynamicLoadOnClientTest.class)); suite.addTest(new JUnit4TestAdapter(GridCacheDynamicLoadOnClientPersistentTest.class)); From e8377164717e9ad0b1a9467bef621b3f358078dc Mon Sep 17 00:00:00 2001 From: devozerov Date: Fri, 28 Dec 2018 17:52:37 +0300 Subject: [PATCH 31/80] WIP on tests. --- .../twostep/JoinPartitionPruningSelfTest.java | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index 5b861a18e1dfb..005c4a146e785 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -17,14 +17,17 @@ package org.apache.ignite.internal.processors.query.h2.twostep; +import org.apache.ignite.Ignite; import org.apache.ignite.IgniteException; import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.managers.communication.GridIoMessage; import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2QueryRequest; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.lang.IgniteInClosure; import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -45,6 +48,42 @@ public class JoinPartitionPruningSelfTest extends GridCommonAbstractTest { /** Parititions tracked during query execution. */ private static final ConcurrentSkipListSet INTERCEPTED_PARTS = new ConcurrentSkipListSet<>(); + /** IP finder. */ + private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder().setShared(true); + + /** Client node name. */ + private static final String CLI_NAME = "cli"; + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + startGrid(getConfiguration("srv1")); + startGrid(getConfiguration("srv2")); + startGrid(getConfiguration("srv3")); + + startGrid(getConfiguration(CLI_NAME).setClientMode(true)); + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + clearIoState(); + + Ignite cli = client(); + + cli.destroyCaches(cli.cacheNames()); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); + } + + /** + * @return Client node. + */ + private IgniteEx client() { + return grid(CLI_NAME); + } + /** * Clear partitions. */ From d829311d5798db40d1fd6d74d4803528509333e6 Mon Sep 17 00:00:00 2001 From: devozerov Date: Fri, 28 Dec 2018 18:31:28 +0300 Subject: [PATCH 32/80] Create table command. --- .../twostep/JoinPartitionPruningSelfTest.java | 177 ++++++++++++++++++ 1 file changed, 177 insertions(+) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index 005c4a146e785..051b98adf3874 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -19,20 +19,26 @@ import org.apache.ignite.Ignite; import org.apache.ignite.IgniteException; +import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.managers.communication.GridIoMessage; +import org.apache.ignite.internal.processors.query.GridQueryProcessor; import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2QueryRequest; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.lang.IgniteInClosure; import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; import java.util.TreeSet; import java.util.concurrent.ConcurrentSkipListSet; import java.util.concurrent.atomic.AtomicInteger; @@ -77,6 +83,113 @@ public class JoinPartitionPruningSelfTest extends GridCommonAbstractTest { stopAllGrids(); } + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String name) throws Exception { + IgniteConfiguration res = super.getConfiguration(name); + + res.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER)); + res.setCommunicationSpi(new TrackingTcpCommunicationSpi()); + + return res; + } + + /** + * Create PARTITIONED table. + * + * @param name Name. + * @param cols Columns. + */ + private void createPartitionedTable(String name, Object... cols) { + createTable0(name, false, cols); + } + + /** + * Create REPLICATED table. + * + * @param name Name. + * @param cols Columns. + */ + private void createReplicatedTable(String name, Object... cols) { + createTable0(name, true, cols); + } + + /** + * Internal CREATE TABLE routine. + * + * @param name Name. + * @param replicated Replicated table flag. + * @param cols Columns. + */ + @SuppressWarnings("StringConcatenationInsideStringBufferAppend") + private void createTable0(String name, boolean replicated, Object... cols) { + List pkCols = new ArrayList<>(); + + String affCol = null; + + StringBuilder sql = new StringBuilder("CREATE TABLE ").append(name).append("("); + + boolean firstCol = true; + + for (Object col : cols) { + Column col0 = col instanceof Column ? (Column)col : new Column((String)col, false, false); + + if (firstCol) + firstCol = true; + else + sql.append(", "); + + sql.append(col0.name()).append(" VARCHAR"); + + if (col0.pk()) + pkCols.add(col0.name()); + + if (col0.affinity()) { + if (affCol != null) + throw new IllegalStateException("Only one affinity column is allowed: " + col0.name()); + + affCol = col0.name(); + } + } + + if (pkCols.isEmpty()) + throw new IllegalStateException("No PKs!"); + + sql.append("PRIMARY KEY ("); + + boolean firstPkCol = true; + + for (String pkCol : pkCols) { + if (firstPkCol) + firstPkCol = false; + else + sql.append(", "); + + sql.append(pkCol); + } + + sql.append(")"); + + sql.append(") WITH \"template=" + (replicated ? "replicated" : "partitioned")); + + if (affCol != null) + sql.append(", affinityKey=" + affCol); + + sql.append("\""); + + execute(sql.toString()); + } + + /** + * Execute SQL query. + * + * @param sql SQL. + */ + private void execute(String sql) { + GridQueryProcessor proc = client().context().query(); + + proc.querySqlFields(new SqlFieldsQuery(sql), false).getAll(); + } + /** * @return Client node. */ @@ -159,4 +272,68 @@ private static class TrackingTcpCommunicationSpi extends TcpCommunicationSpi { super.sendMessage(node, msg, ackC); } } + + /** + * @param name Name. + * @return PK column. + */ + public Column pkColumn(String name) { + return new Column(name, true, false); + } + + /** + * @param name Name. + * @return Affintiy column. + */ + public Column affinityColumn(String name) { + return new Column(name, true, true); + } + + /** + * Column. + */ + private static class Column { + /** Name. */ + private final String name; + + /** PK. */ + private final boolean pk; + + /** Affinity key. */ + private final boolean aff; + + /** + * Constructor. + * + * @param name Name. + * @param pk PK flag. + * @param aff Affinity flag. + */ + public Column(String name, boolean pk, boolean aff) { + this.name = name; + this.pk = pk; + this.aff = aff; + } + + /** + * @return Name. + */ + public String name() { + return name; + } + + /** + * @return PK flag. + */ + public boolean pk() { + return pk; + } + + /** + * @return Affintiy flag. + */ + public boolean affinity() { + return aff; + } + } } From 44691700a51e333ee5b7feb1e65855ffb8307d11 Mon Sep 17 00:00:00 2001 From: devozerov Date: Fri, 28 Dec 2018 18:41:38 +0300 Subject: [PATCH 33/80] Infrastructure finished. --- .../twostep/JoinPartitionPruningSelfTest.java | 46 +++++++++++++++---- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index 051b98adf3874..386825dcd7b37 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -33,6 +33,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -90,9 +91,32 @@ public class JoinPartitionPruningSelfTest extends GridCommonAbstractTest { res.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER)); res.setCommunicationSpi(new TrackingTcpCommunicationSpi()); + res.setLocalHost("127.0.0.1"); + return res; } + /** + * Test simple join. + */ + @Test + public void testSimpleJoin() { + createPartitionedTable("t1", + pkColumn("k1"), + "v2"); + + createPartitionedTable("t2", + pkColumn("k1"), + affinityColumn("ak2"), + "v3"); + + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.ak2 = 1"); + + assertPartitions( + parititon("t2", 1) + ); + } + /** * Create PARTITIONED table. * @@ -127,18 +151,10 @@ private void createTable0(String name, boolean replicated, Object... cols) { String affCol = null; StringBuilder sql = new StringBuilder("CREATE TABLE ").append(name).append("("); - - boolean firstCol = true; - for (Object col : cols) { Column col0 = col instanceof Column ? (Column)col : new Column((String)col, false, false); - if (firstCol) - firstCol = true; - else - sql.append(", "); - - sql.append(col0.name()).append(" VARCHAR"); + sql.append(col0.name()).append(" VARCHAR, "); if (col0.pk()) pkCols.add(col0.name()); @@ -170,9 +186,10 @@ private void createTable0(String name, boolean replicated, Object... cols) { sql.append(")"); sql.append(") WITH \"template=" + (replicated ? "replicated" : "partitioned")); + sql.append(", CACHE_NAME=" + name); if (affCol != null) - sql.append(", affinityKey=" + affCol); + sql.append(", AFFINITY_KEY=" + affCol); sql.append("\""); @@ -247,6 +264,15 @@ private static void assertNoRequests() { assertEquals("Requests were sent: " + INTERCEPTED_REQS.get(), 0, INTERCEPTED_REQS.get()); } + /** + * @param cacheName Cache name. + * @param key Key. + * @return Partition. + */ + private int parititon(String cacheName, Object key) { + return client().affinity(cacheName).partition(key); + } + /** * TCP communication SPI which will track outgoing query requests. */ From eabd8b8c4fa29104d0d9fa3ea71a7df9b5ff6c4c Mon Sep 17 00:00:00 2001 From: devozerov Date: Fri, 28 Dec 2018 18:45:31 +0300 Subject: [PATCH 34/80] WIP. --- .../query/h2/affinity/join/PartitionTableModel.java | 2 +- .../h2/twostep/JoinPartitionPruningSelfTest.java | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java index 6d11ee96a90f6..afcd6cead7a92 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java @@ -67,7 +67,7 @@ public void addTable(PartitionJoinTable tbl, PartitionJoinAffinityDescriptor aff @Nullable public PartitionJoinTable table(String alias) { PartitionJoinTable res = tbls.get(alias); - assert res != null || excludedTblNames.contains(alias); + assert res != null || (excludedTblNames != null && excludedTblNames.contains(alias)); return res; } diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index 386825dcd7b37..1b5beb12d02d9 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -110,10 +110,19 @@ public void testSimpleJoin() { affinityColumn("ak2"), "v3"); - execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.ak2 = 1"); +// execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.ak2 = 1"); +// +// assertPartitions( +// parititon("t2", 1) +// ); +// clearIoState(); + + // TODO: Doesn't work. + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.ak2 = 1 OR t1.k1 = 2"); assertPartitions( - parititon("t2", 1) + parititon("t2", 1), + parititon("t1", 2) ); } From b8370558561bb0607361a1c376eeb2b891801fd1 Mon Sep 17 00:00:00 2001 From: devozerov Date: Fri, 28 Dec 2018 18:52:43 +0300 Subject: [PATCH 35/80] Fixes. --- .../processors/query/h2/affinity/PartitionExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java index a3b8d8c61ae99..6cb0a8b0997ba 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java @@ -174,7 +174,7 @@ private List prepareTableModel0(GridSqlAst from, PartitionTa GridSqlJoin join = (GridSqlJoin)from; List leftTbls = prepareTableModel0(join.leftTable(), model); - List rightTbls = prepareTableModel0(join.leftTable(), model); + List rightTbls = prepareTableModel0(join.rightTable(), model); if (join.isLeftOuter()) { // "a LEFT JOIN b" is transformed into "a", and "b" is put into special stop-list. From f11f0bc8516594acece49ff5d051185e5d238466 Mon Sep 17 00:00:00 2001 From: devozerov Date: Sat, 29 Dec 2018 11:35:33 +0300 Subject: [PATCH 36/80] Fixed node filter. --- .../processors/query/h2/affinity/PartitionExtractor.java | 4 ++-- .../query/h2/affinity/PartitionExtractorUtils.java | 7 +++++-- .../query/h2/affinity/join/PartitionTableModel.java | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java index 6cb0a8b0997ba..fd7197baa7304 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java @@ -189,7 +189,7 @@ private List prepareTableModel0(GridSqlAst from, PartitionTa // cross join here, real join condition will be found in WHERE clause later. PartitionJoinCondition cond = PartitionExtractorUtils.parseJoinCondition(join.on()); - if (cond != null && cond.cross()) + if (cond != null && !cond.cross()) model.addJoin(cond); ArrayList res = new ArrayList<>(leftTbls.size() + rightTbls.size()); @@ -383,7 +383,7 @@ else if (right instanceof GridSqlColumn) { if (!disjunct) { PartitionJoinCondition cond = PartitionExtractorUtils.parseJoinCondition(op); - if (cond != null && cond.cross()) + if (cond != null && !cond.cross()) tblModel.addJoin(cond); } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java index 2b064f71b63e3..0f516bd72c60f 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java @@ -170,17 +170,20 @@ public static PartitionJoinCondition parseJoinCondition(GridSqlElement on) { * @return Affinity identifier. */ public static PartitionJoinAffinityDescriptor affinityForCache(CacheConfiguration ccfg) { - // Partition could be extracted only from PARTITIONED cache. + // Partition could be extracted only from PARTITIONED caches. if (ccfg.getCacheMode() != CacheMode.PARTITIONED) return null; PartitionAffinityFunctionType aff = ccfg.getAffinity().getClass().equals(RendezvousAffinityFunction.class) ? PartitionAffinityFunctionType.RENDEZVOUS : PartitionAffinityFunctionType.CUSTOM; + boolean hasNodeFilter = ccfg.getNodeFilter() != null && + !(ccfg.getNodeFilter() instanceof CacheConfiguration.IgniteAllNodesPredicate); + return new PartitionJoinAffinityDescriptor( aff, ccfg.getAffinity().partitions(), - ccfg.getNodeFilter() != null + hasNodeFilter ); } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java index afcd6cead7a92..1552f785d4dac 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java @@ -112,7 +112,7 @@ public void addJoin(PartitionJoinCondition cond) { return; // At least one column in condition is not affinity column, return. - if (leftTbl.isAffinityColumn(cond.leftColumn()) && rightTbl.isAffinityColumn(cond.rightColumn())) + if (!leftTbl.isAffinityColumn(cond.leftColumn()) || !rightTbl.isAffinityColumn(cond.rightColumn())) return; PartitionJoinGroup leftGrp = grps.get(leftTbl.joinGroup()); From 3dedf90687820588ac74efe1d4bd9eed6120e182 Mon Sep 17 00:00:00 2001 From: devozerov Date: Sat, 29 Dec 2018 11:44:27 +0300 Subject: [PATCH 37/80] Complex join worked! --- .../query/h2/affinity/join/PartitionTableModel.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java index 1552f785d4dac..9fa5fdcde749f 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java @@ -115,8 +115,11 @@ public void addJoin(PartitionJoinCondition cond) { if (!leftTbl.isAffinityColumn(cond.leftColumn()) || !rightTbl.isAffinityColumn(cond.rightColumn())) return; + // Remember join group of the right table as it will be changed below. + int rightGrpId = rightTbl.joinGroup(); + PartitionJoinGroup leftGrp = grps.get(leftTbl.joinGroup()); - PartitionJoinGroup rightGrp = grps.get(rightTbl.joinGroup()); + PartitionJoinGroup rightGrp = grps.get(rightGrpId); assert leftGrp != null; assert rightGrp != null; @@ -132,7 +135,7 @@ public void addJoin(PartitionJoinCondition cond) { leftGrp.addTable(tbl); } - grps.remove(rightTbl.joinGroup()); + grps.remove(rightGrpId); } /** From aa53788221b22b77eb57744af6ec9d920e6622a4 Mon Sep 17 00:00:00 2001 From: devozerov Date: Sat, 29 Dec 2018 11:48:17 +0300 Subject: [PATCH 38/80] WIP on tests. --- .../twostep/JoinPartitionPruningSelfTest.java | 46 +++++++++++++++---- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index 1b5beb12d02d9..dc8055877673d 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -110,14 +110,33 @@ public void testSimpleJoin() { affinityColumn("ak2"), "v3"); -// execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.ak2 = 1"); -// -// assertPartitions( -// parititon("t2", 1) -// ); -// clearIoState(); - - // TODO: Doesn't work. + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = 1"); + + assertPartitionsAndClear( + parititon("t1", 1) + ); + + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1._KEY = 2"); + + assertPartitionsAndClear( + parititon("t2", 2) + ); + } + + /** + * Test simple join. + */ + @Test + public void testComplexJoin() { + createPartitionedTable("t1", + pkColumn("k1"), + "v2"); + + createPartitionedTable("t2", + pkColumn("k1"), + affinityColumn("ak2"), + "v3"); + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.ak2 = 1 OR t1.k1 = 2"); assertPartitions( @@ -231,6 +250,17 @@ private static void clearIoState() { INTERCEPTED_PARTS.clear(); } + /** + * Make sure that expected partitions are logged, then clear IO state. + * + * @param expParts Expected partitions. + */ + private static void assertPartitionsAndClear(int... expParts) { + assertPartitions(expParts); + + clearIoState(); + } + /** * Make sure that expected partitions are logged. * From aeec43698d18e9a5e3adf7a3530769cfaa5e6403 Mon Sep 17 00:00:00 2001 From: devozerov Date: Sat, 29 Dec 2018 11:55:42 +0300 Subject: [PATCH 39/80] WIP on tests. --- .../twostep/JoinPartitionPruningSelfTest.java | 40 +++++++++++++++++-- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index dc8055877673d..f58f688605da7 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -110,16 +110,36 @@ public void testSimpleJoin() { affinityColumn("ak2"), "v3"); + // Key (no alias). execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = 1"); assertPartitionsAndClear( parititon("t1", 1) ); + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ?", 1); + + assertPartitionsAndClear( + parititon("t1", 1) + ); + + // Key (alias). execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1._KEY = 2"); assertPartitionsAndClear( - parititon("t2", 2) + parititon("t1", 2) + ); + + // Non-affinity key. + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.k1 = 4"); + + assertNoPartitions(); + + // Affinity key. + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.ak2 = 3"); + + assertPartitionsAndClear( + parititon("t2", 3) ); } @@ -229,10 +249,13 @@ private void createTable0(String name, boolean replicated, Object... cols) { * * @param sql SQL. */ - private void execute(String sql) { - GridQueryProcessor proc = client().context().query(); + private List> execute(String sql, Object... args) { + SqlFieldsQuery qry = new SqlFieldsQuery(sql); + + if (args != null && args.length > 0) + qry.setArgs(args); - proc.querySqlFields(new SqlFieldsQuery(sql), false).getAll(); + return client().context().query().querySqlFields(qry, false).getAll(); } /** @@ -288,6 +311,15 @@ private static void assertPartitions(Collection expParts) { expParts0, actualParts); } + /** + * Make sure that no partitions were extracted, then clear IO state. + */ + private static void assertNoPartitionsAndClear() { + assertNoPartitions(); + + clearIoState(); + } + /** * Make sure that no partitions were extracted. */ From 047c0291b8b02c26900078317f13db8af48eb5a3 Mon Sep 17 00:00:00 2001 From: devozerov Date: Sat, 29 Dec 2018 11:57:57 +0300 Subject: [PATCH 40/80] WIP. --- .../query/h2/twostep/JoinPartitionPruningSelfTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index f58f688605da7..3ddc9159e4cb0 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -24,7 +24,6 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.managers.communication.GridIoMessage; -import org.apache.ignite.internal.processors.query.GridQueryProcessor; import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2QueryRequest; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.lang.IgniteInClosure; @@ -141,6 +140,9 @@ public void testSimpleJoin() { assertPartitionsAndClear( parititon("t2", 3) ); + + // Complex key. + // TODO } /** From 34a5941de296a56f5c1cf3406bddb77331476d52 Mon Sep 17 00:00:00 2001 From: devozerov Date: Sat, 29 Dec 2018 12:02:01 +0300 Subject: [PATCH 41/80] Fixing conversions. --- .../query/h2/affinity/PartitionExtractor.java | 5 ++++- .../h2/twostep/JoinPartitionPruningSelfTest.java | 12 ++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java index fd7197baa7304..2064e04866abd 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java @@ -19,6 +19,7 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery; +import org.apache.ignite.internal.processors.query.h2.H2Utils; import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing; import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinAffinityDescriptor; import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinCondition; @@ -431,7 +432,9 @@ else if (right instanceof GridSqlColumn) { return null; if (rightConst != null) { - int part = idx.kernalContext().affinity().partition(tbl.cacheName(), rightConst.value().getObject()); + Object constVal = H2Utils.convert(rightConst.value().getObject(), idx, leftCol0.getType()); + + int part = idx.kernalContext().affinity().partition(tbl.cacheName(), constVal); return new PartitionConstantNode(tbl0, part); } diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index 3ddc9159e4cb0..d631ecdafa95f 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -113,20 +113,20 @@ public void testSimpleJoin() { execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = 1"); assertPartitionsAndClear( - parititon("t1", 1) + parititon("t1", "1") ); execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ?", 1); assertPartitionsAndClear( - parititon("t1", 1) + parititon("t1", "1") ); // Key (alias). execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1._KEY = 2"); assertPartitionsAndClear( - parititon("t1", 2) + parititon("t1", "2") ); // Non-affinity key. @@ -138,7 +138,7 @@ public void testSimpleJoin() { execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.ak2 = 3"); assertPartitionsAndClear( - parititon("t2", 3) + parititon("t2", "3") ); // Complex key. @@ -162,8 +162,8 @@ public void testComplexJoin() { execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.ak2 = 1 OR t1.k1 = 2"); assertPartitions( - parititon("t2", 1), - parititon("t1", 2) + parititon("t2", "1"), + parititon("t1", "2") ); } From cd7e8917020f08ef9e7e02b64e34341d76371edd Mon Sep 17 00:00:00 2001 From: devozerov Date: Sat, 29 Dec 2018 12:23:05 +0300 Subject: [PATCH 42/80] Simple join test. --- .../twostep/JoinPartitionPruningSelfTest.java | 79 +++++++++++++++++-- 1 file changed, 71 insertions(+), 8 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index d631ecdafa95f..5b0e6c7e920d2 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -19,6 +19,7 @@ import org.apache.ignite.Ignite; import org.apache.ignite.IgniteException; +import org.apache.ignite.binary.BinaryObject; import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.IgniteConfiguration; @@ -109,40 +110,100 @@ public void testSimpleJoin() { affinityColumn("ak2"), "v3"); + execute("INSERT INTO t1 VALUES ('1', '1')"); + execute("INSERT INTO t2 VALUES ('1', '1', '1')"); + + execute("INSERT INTO t1 VALUES ('2', '2')"); + execute("INSERT INTO t2 VALUES ('2', '2', '2')"); + + execute("INSERT INTO t1 VALUES ('3', '3')"); + execute("INSERT INTO t2 VALUES ('3', '3', '3')"); + + execute("INSERT INTO t1 VALUES ('4', '4')"); + execute("INSERT INTO t2 VALUES ('4', '4', '4')"); + + execute("INSERT INTO t1 VALUES ('5', '5')"); + execute("INSERT INTO t2 VALUES ('5', '5', '5')"); + // Key (no alias). - execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = 1"); + List> res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = 1"); + assertPartitionsAndClear( + parititon("t1", "1") + ); + assertEquals(1, res.size()); + assertEquals("1", res.get(0).get(0)); + res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = '1'"); assertPartitionsAndClear( parititon("t1", "1") ); + assertEquals(1, res.size()); + assertEquals("1", res.get(0).get(0)); - execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ?", 1); + res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ?", 1); + assertPartitionsAndClear( + parititon("t1", "1") + ); + assertEquals(1, res.size()); + assertEquals("1", res.get(0).get(0)); + res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ?", "1"); assertPartitionsAndClear( parititon("t1", "1") ); + assertEquals(1, res.size()); + assertEquals("1", res.get(0).get(0)); // Key (alias). - execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1._KEY = 2"); + res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1._KEY = '2'"); + assertPartitionsAndClear( + parititon("t1", "2") + ); + assertEquals(1, res.size()); + assertEquals("2", res.get(0).get(0)); + res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1._KEY = ?", "2"); assertPartitionsAndClear( parititon("t1", "2") ); + assertEquals(1, res.size()); + assertEquals("2", res.get(0).get(0)); // Non-affinity key. - execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.k1 = 4"); + res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.k1 = 3"); + assertNoPartitions(); + assertEquals(1, res.size()); + assertEquals("3", res.get(0).get(0)); + res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.k1 = ?", "3"); assertNoPartitions(); + assertEquals(1, res.size()); + assertEquals("3", res.get(0).get(0)); // Affinity key. - execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.ak2 = 3"); + res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.ak2 = 4"); + assertPartitionsAndClear( + parititon("t2", "4") + ); + assertEquals(1, res.size()); + assertEquals("4", res.get(0).get(0)); + res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.ak2 = ?", "4"); assertPartitionsAndClear( - parititon("t2", "3") + parititon("t2", "4") ); + assertEquals(1, res.size()); + assertEquals("4", res.get(0).get(0)); // Complex key. - // TODO + BinaryObject key = client().binary().builder("t2_key").setField("k1", "5").setField("ak2", "5").build(); + + res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2._KEY = ?", key); + assertPartitionsAndClear( + parititon("t2", "5") + ); + assertEquals(1, res.size()); + assertEquals("5", res.get(0).get(0)); } /** @@ -238,8 +299,10 @@ private void createTable0(String name, boolean replicated, Object... cols) { sql.append(") WITH \"template=" + (replicated ? "replicated" : "partitioned")); sql.append(", CACHE_NAME=" + name); - if (affCol != null) + if (affCol != null) { sql.append(", AFFINITY_KEY=" + affCol); + sql.append(", KEY_TYPE=" + name + "_key"); + } sql.append("\""); From 3a0dcf379775621ac1b195abaa5567f08b37688a Mon Sep 17 00:00:00 2001 From: devozerov Date: Sat, 29 Dec 2018 12:37:35 +0300 Subject: [PATCH 43/80] Cross-join tests. --- .../twostep/JoinPartitionPruningSelfTest.java | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index 5b0e6c7e920d2..de18bfd887bd8 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -211,6 +211,7 @@ public void testSimpleJoin() { */ @Test public void testComplexJoin() { + // TODO createPartitionedTable("t1", pkColumn("k1"), "v2"); @@ -228,6 +229,75 @@ public void testComplexJoin() { ); } + /** + * Test cross-joins. They cannot "transfer" partitions between joined tables. + */ + @Test + public void testCrossJoin() { + createPartitionedTable("t1", + pkColumn("k1"), + "v2"); + + createPartitionedTable("t2", + pkColumn("k1"), + affinityColumn("ak2"), + "v3"); + + execute("INSERT INTO t1 VALUES ('1', '1')"); + execute("INSERT INTO t2 VALUES ('1', '1', '1')"); + + execute("INSERT INTO t1 VALUES ('2', '2')"); + execute("INSERT INTO t2 VALUES ('2', '2', '2')"); + + execute("INSERT INTO t1 VALUES ('3', '3')"); + execute("INSERT INTO t2 VALUES ('3', '3', '3')"); + + // Left table, should work. + List> res = execute("SELECT * FROM t1, t2 WHERE t1.k1 = '1'"); + assertPartitionsAndClear( + parititon("t1", "1") + ); + assertEquals(1, res.size()); + assertEquals("1", res.get(0).get(0)); + + res = execute("SELECT * FROM t1 INNER JOIN t2 ON 1=1 WHERE t1.k1 = '1'"); + assertPartitionsAndClear( + parititon("t1", "1") + ); + assertEquals(1, res.size()); + assertEquals("1", res.get(0).get(0)); + + // Right table, should work. + res = execute("SELECT * FROM t1, t2 WHERE t2.ak2 = '2'"); + assertPartitionsAndClear( + parititon("t2", "2") + ); + assertEquals(1, res.size()); + assertEquals("2", res.get(0).get(0)); + + res = execute("SELECT * FROM t1 INNER JOIN t2 ON 1=1 WHERE t2.ak2 = '2'"); + assertPartitionsAndClear( + parititon("t2", "2") + ); + assertEquals(1, res.size()); + assertEquals("2", res.get(0).get(0)); + + // Two tables, should not work. + res = execute("SELECT * FROM t1, t2 WHERE t1.k1='3' AND t2.ak2 = '3'"); + assertNoPartitions(); + + res = execute("SELECT * FROM t1, t2 WHERE t1.k1='3' OR t2.ak2 = '3'"); + assertNoPartitions(); + } + + /** + * Test non-equijoins. + */ + @Test + public void testThetaJoin() { + // TODO + } + /** * Create PARTITIONED table. * From fd999fe0aed006920cde16659aac9f429a8abce7 Mon Sep 17 00:00:00 2001 From: devozerov Date: Sat, 29 Dec 2018 12:49:21 +0300 Subject: [PATCH 44/80] WIP on tests. --- .../twostep/JoinPartitionPruningSelfTest.java | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index de18bfd887bd8..59c7c4b28f02c 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -206,6 +206,14 @@ public void testSimpleJoin() { assertEquals("5", res.get(0).get(0)); } + /** + * Test various expressions. + */ + @Test + public void testExpressions() { + // TODO + } + /** * Test simple join. */ @@ -298,6 +306,53 @@ public void testThetaJoin() { // TODO } + /** + * Test joins with REPLICTED caches. + */ + @Test + public void testJoinWithReplicated() { + // TODO + } + + /** + * Test joins with different affinity functions. + */ + @Test + public void testJoinWithDifferentAffinityFunctions() { + // Partition count. + // TODO + + // Different affinity functions. + // TODO + + // Node filters. + // TODO + } + + /** + * Test joins with subqueries. + */ + @Test + public void testJoinWithSubquery() { + // TODO + } + + /** + * Test joins when explicit partitions are set. + */ + @Test + public void testExplicitParititons() { + // TODO + } + + /** + * Test outer joins. + */ + @Test + public void testOuterJoin() { + // TODO + } + /** * Create PARTITIONED table. * From 4bb5840c3ec491e5be26ad417ec64ac703fdf6cd Mon Sep 17 00:00:00 2001 From: devozerov Date: Sat, 29 Dec 2018 12:55:21 +0300 Subject: [PATCH 45/80] Refactoring. --- .../PartitionAffinityFunctionType.java | 2 +- .../query/h2/affinity/PartitionAllNode.java | 1 - .../h2/affinity/PartitionConstantNode.java | 3 +- .../query/h2/affinity/PartitionExtractor.java | 188 +++++++++++++++-- .../h2/affinity/PartitionExtractorUtils.java | 196 ------------------ .../{join => }/PartitionJoinCondition.java | 2 +- .../{join => }/PartitionJoinGroup.java | 18 +- .../query/h2/affinity/PartitionNoneNode.java | 1 - .../h2/affinity/PartitionParameterNode.java | 3 +- .../query/h2/affinity/PartitionResult.java | 7 +- .../h2/affinity/PartitionSingleNode.java | 5 +- ...tionJoinTable.java => PartitionTable.java} | 8 +- ... => PartitionTableAffinityDescriptor.java} | 11 +- .../{join => }/PartitionTableModel.java | 20 +- .../twostep/JoinPartitionPruningSelfTest.java | 8 + 15 files changed, 212 insertions(+), 261 deletions(-) rename modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/{join => }/PartitionAffinityFunctionType.java (96%) delete mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java rename modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/{join => }/PartitionJoinCondition.java (98%) rename modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/{join => }/PartitionJoinGroup.java (74%) rename modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/{join/PartitionJoinTable.java => PartitionTable.java} (94%) rename modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/{join/PartitionJoinAffinityDescriptor.java => PartitionTableAffinityDescriptor.java} (88%) rename modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/{join => }/PartitionTableModel.java (86%) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionAffinityFunctionType.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionAffinityFunctionType.java similarity index 96% rename from modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionAffinityFunctionType.java rename to modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionAffinityFunctionType.java index 58f62abf6e83a..4ddafff55c73f 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionAffinityFunctionType.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionAffinityFunctionType.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.internal.processors.query.h2.affinity.join; +package org.apache.ignite.internal.processors.query.h2.affinity; import org.jetbrains.annotations.Nullable; diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionAllNode.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionAllNode.java index f8110182e7aa5..30860f55376b7 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionAllNode.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionAllNode.java @@ -17,7 +17,6 @@ package org.apache.ignite.internal.processors.query.h2.affinity; -import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionTableModel; import org.apache.ignite.internal.util.typedef.internal.S; import java.util.Collection; diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionConstantNode.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionConstantNode.java index aa04e4ee747f8..9e258aec73e99 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionConstantNode.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionConstantNode.java @@ -17,7 +17,6 @@ package org.apache.ignite.internal.processors.query.h2.affinity; -import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinTable; import org.apache.ignite.internal.util.typedef.internal.S; /** @@ -33,7 +32,7 @@ public class PartitionConstantNode extends PartitionSingleNode { * @param tbl Table. * @param part Partition. */ - public PartitionConstantNode(PartitionJoinTable tbl, int part) { + public PartitionConstantNode(PartitionTable tbl, int part) { super(tbl); this.part = part; diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java index 2064e04866abd..cfeb1a5810ac1 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java @@ -18,13 +18,12 @@ package org.apache.ignite.internal.processors.query.h2.affinity; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery; import org.apache.ignite.internal.processors.query.h2.H2Utils; import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing; -import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinAffinityDescriptor; -import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinCondition; -import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionTableModel; -import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinTable; import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlAlias; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlAst; @@ -37,6 +36,7 @@ import org.apache.ignite.internal.processors.query.h2.sql.GridSqlParameter; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQuery; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlSelect; +import org.apache.ignite.internal.processors.query.h2.sql.GridSqlTable; import org.h2.table.Column; import org.jetbrains.annotations.Nullable; @@ -102,7 +102,7 @@ public PartitionResult extract(GridSqlQuery qry) throws IgniteCheckedException { @SuppressWarnings("IfMayBeConditional") public PartitionResult mergeMapQueries(List qrys) { // Check if merge is possible. - PartitionJoinAffinityDescriptor aff = null; + PartitionTableAffinityDescriptor aff = null; for (GridCacheSqlQuery qry : qrys) { PartitionResult qryRes = (PartitionResult)qry.derivedPartitions(); @@ -169,18 +169,18 @@ private PartitionTableModel prepareTableModel(GridSqlAst from) { * @param model Table model. * @return {@code True} if extracted tables successfully, {@code false} if failed to extract. */ - private List prepareTableModel0(GridSqlAst from, PartitionTableModel model) { + private List prepareTableModel0(GridSqlAst from, PartitionTableModel model) { if (from instanceof GridSqlJoin) { // Process JOIN recursively. GridSqlJoin join = (GridSqlJoin)from; - List leftTbls = prepareTableModel0(join.leftTable(), model); - List rightTbls = prepareTableModel0(join.rightTable(), model); + List leftTbls = prepareTableModel0(join.leftTable(), model); + List rightTbls = prepareTableModel0(join.rightTable(), model); if (join.isLeftOuter()) { // "a LEFT JOIN b" is transformed into "a", and "b" is put into special stop-list. // If a condition is met on "b" afterwards, we will stop partition pruning process. - for (PartitionJoinTable rightTbl : rightTbls) + for (PartitionTable rightTbl : rightTbls) model.addExcludedTable(rightTbl.alias()); return leftTbls; @@ -188,12 +188,12 @@ private List prepareTableModel0(GridSqlAst from, PartitionTa // Extract equi-join or cross-join from condition. For normal INNER JOINs most likely we will have "1=1" // cross join here, real join condition will be found in WHERE clause later. - PartitionJoinCondition cond = PartitionExtractorUtils.parseJoinCondition(join.on()); + PartitionJoinCondition cond = parseJoinCondition(join.on()); if (cond != null && !cond.cross()) model.addJoin(cond); - ArrayList res = new ArrayList<>(leftTbls.size() + rightTbls.size()); + ArrayList res = new ArrayList<>(leftTbls.size() + rightTbls.size()); res.addAll(leftTbls); res.addAll(rightTbls); @@ -201,11 +201,157 @@ private List prepareTableModel0(GridSqlAst from, PartitionTa return res; } - PartitionJoinTable tbl = PartitionExtractorUtils.prepareTable(from, model); + PartitionTable tbl = prepareTable(from, model); return Collections.singletonList(tbl); } + /** + * Try parsing condition as simple JOIN codition. Only equijoins are supported for now, so anything more complex + * than "A.a = B.b" are not processed. + * + * @param on Initial AST. + * @return Join condition or {@code null} if not simple equijoin. + */ + private static PartitionJoinCondition parseJoinCondition(GridSqlElement on) { + if (on instanceof GridSqlOperation) { + GridSqlOperation on0 = (GridSqlOperation)on; + + if (on0.operationType() == GridSqlOperationType.EQUAL) { + // Check for cross-join first. + GridSqlConst leftConst = unwrapConst(on0.child(0)); + GridSqlConst rightConst = unwrapConst(on0.child(1)); + + if (leftConst != null && rightConst != null) { + try { + int leftConstval = leftConst.value().getInt(); + int rightConstVal = rightConst.value().getInt(); + + if (leftConstval == rightConstVal) + return PartitionJoinCondition.CROSS; + } + catch (Exception ignore) { + // No-op. + } + } + + // This is not cross-join, neither normal join between columns. + if (leftConst != null || rightConst != null) + return null; + + // Check for normal equi-join. + GridSqlColumn left = unwrapColumn(on0.child(0)); + GridSqlColumn right = unwrapColumn(on0.child(1)); + + if (left != null && right != null) { + String leftAlias = left.tableAlias(); + String rightAlias = right.tableAlias(); + + String leftCol = left.columnName(); + String rightCol = right.columnName(); + + return new PartitionJoinCondition(leftAlias, rightAlias, leftCol, rightCol); + } + } + } + + return null; + } + + /** + * Prepare single table. + * + * @param from Expression. + * @param tblModel Table model. + * @return Added table or {@code null} if table is exlcuded from the model. + */ + private static PartitionTable prepareTable(GridSqlAst from, PartitionTableModel tblModel) { + // Unwrap alias. We assume that every table must be aliased. + assert from instanceof GridSqlAlias; + + String alias = ((GridSqlAlias)from).alias(); + + from = from.child(); + + if (from instanceof GridSqlTable) { + // Normal table. + GridSqlTable from0 = (GridSqlTable)from; + + GridH2Table tbl0 = from0.dataTable(); + + // Unknown table type, e.g. temp table. + if (tbl0 == null) { + tblModel.addExcludedTable(alias); + + return null; + } + + String cacheName = tbl0.cacheName(); + + String affColName = null; + String secondAffColName = null; + + for (Column col : tbl0.getColumns()) { + if (tbl0.isColumnForPartitionPruningStrict(col)) { + if (affColName == null) + affColName = col.getName(); + else { + secondAffColName = col.getName(); + + // Break as we cannot have more than two affinity key columns. + break; + } + } + } + + PartitionTable tbl = new PartitionTable(alias, cacheName, affColName, secondAffColName); + PartitionTableAffinityDescriptor aff = affinityForCache(tbl0.cacheInfo().config()); + + if (aff == null) { + // Non-standard affinity, exclude table. + tblModel.addExcludedTable(alias); + + return null; + } + + tblModel.addTable(tbl, aff); + + return tbl; + } + else { + // Subquery/union/view, etc. + assert alias != null; + + tblModel.addExcludedTable(alias); + + return null; + } + } + + /** + * Prepare affinity identifier for cache. + * + * @param ccfg Cache configuration. + * @return Affinity identifier. + */ + private static PartitionTableAffinityDescriptor affinityForCache(CacheConfiguration ccfg) { + // Partition could be extracted only from PARTITIONED caches. + if (ccfg.getCacheMode() != CacheMode.PARTITIONED) + return null; + + PartitionAffinityFunctionType aff = ccfg.getAffinity().getClass().equals(RendezvousAffinityFunction.class) ? + PartitionAffinityFunctionType.RENDEZVOUS : PartitionAffinityFunctionType.CUSTOM; + + boolean hasNodeFilter = ccfg.getNodeFilter() != null && + !(ccfg.getNodeFilter() instanceof CacheConfiguration.IgniteAllNodesPredicate); + + return new PartitionTableAffinityDescriptor( + aff, + ccfg.getAffinity().partitions(), + hasNodeFilter + ); + } + /** * Extract partitions from expression. * @@ -380,18 +526,18 @@ else if (right instanceof GridSqlParameter) { rightConst = null; rightParam = (GridSqlParameter)right; } - else if (right instanceof GridSqlColumn) { - if (!disjunct) { - PartitionJoinCondition cond = PartitionExtractorUtils.parseJoinCondition(op); + else { + if (right instanceof GridSqlColumn) { + if (!disjunct) { + PartitionJoinCondition cond = parseJoinCondition(op); - if (cond != null && !cond.cross()) - tblModel.addJoin(cond); - } + if (cond != null && !cond.cross()) + tblModel.addJoin(cond); + } + } return PartitionAllNode.INSTANCE; } - else - return PartitionAllNode.INSTANCE; PartitionSingleNode part = extractSingle(leftCol, rightConst, rightParam, tblModel); @@ -425,7 +571,7 @@ else if (right instanceof GridSqlColumn) { if (!tbl.isColumnForPartitionPruning(leftCol0)) return null; - PartitionJoinTable tbl0 = tblModel.table(leftCol.tableAlias()); + PartitionTable tbl0 = tblModel.table(leftCol.tableAlias()); // If table is in ignored set, then we cannot use it for partition extraction. if (tbl0 == null) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java deleted file mode 100644 index 0f516bd72c60f..0000000000000 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractorUtils.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.query.h2.affinity; - -import org.apache.ignite.cache.CacheMode; -import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; -import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionAffinityFunctionType; -import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinAffinityDescriptor; -import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinCondition; -import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinTable; -import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionTableModel; -import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table; -import org.apache.ignite.internal.processors.query.h2.sql.GridSqlAlias; -import org.apache.ignite.internal.processors.query.h2.sql.GridSqlAst; -import org.apache.ignite.internal.processors.query.h2.sql.GridSqlColumn; -import org.apache.ignite.internal.processors.query.h2.sql.GridSqlConst; -import org.apache.ignite.internal.processors.query.h2.sql.GridSqlElement; -import org.apache.ignite.internal.processors.query.h2.sql.GridSqlOperation; -import org.apache.ignite.internal.processors.query.h2.sql.GridSqlOperationType; -import org.apache.ignite.internal.processors.query.h2.sql.GridSqlTable; -import org.h2.table.Column; - -/** - * Utility methods for partition extraction. - */ -public class PartitionExtractorUtils { - - /** - * Prepare single table. - * - * @param from Expression. - * @param tblModel Table model. - * @return Added table or {@code null} if table is exlcuded from the model. - */ - public static PartitionJoinTable prepareTable(GridSqlAst from, PartitionTableModel tblModel) { - // Unwrap alias. We assume that every table must be aliased. - assert from instanceof GridSqlAlias; - - String alias = ((GridSqlAlias)from).alias(); - - from = from.child(); - - if (from instanceof GridSqlTable) { - // Normal table. - GridSqlTable from0 = (GridSqlTable)from; - - GridH2Table tbl0 = from0.dataTable(); - - // Unknown table type, e.g. temp table. - if (tbl0 == null) { - tblModel.addExcludedTable(alias); - - return null; - } - - String cacheName = tbl0.cacheName(); - - String affColName = null; - String secondAffColName = null; - - for (Column col : tbl0.getColumns()) { - if (tbl0.isColumnForPartitionPruningStrict(col)) { - if (affColName == null) - affColName = col.getName(); - else { - secondAffColName = col.getName(); - - // Break as we cannot have more than two affinity key columns. - break; - } - } - } - - PartitionJoinTable tbl = new PartitionJoinTable(alias, cacheName, affColName, secondAffColName); - PartitionJoinAffinityDescriptor aff = affinityForCache(tbl0.cacheInfo().config()); - - if (aff == null) { - // Non-standard affinity, exclude table. - tblModel.addExcludedTable(alias); - - return null; - } - - tblModel.addTable(tbl, aff); - - return tbl; - } - else { - // Subquery/union/view, etc. - assert alias != null; - - tblModel.addExcludedTable(alias); - - return null; - } - } - - /** - * Try parsing condition as simple JOIN codition. Only equijoins are supported for now, so anything more complex - * than "A.a = B.b" are not processed. - * - * @param on Initial AST. - * @return Join condition or {@code null} if not simple equijoin. - */ - public static PartitionJoinCondition parseJoinCondition(GridSqlElement on) { - if (on instanceof GridSqlOperation) { - GridSqlOperation on0 = (GridSqlOperation)on; - - if (on0.operationType() == GridSqlOperationType.EQUAL) { - // Check for cross-join first. - GridSqlConst leftConst = PartitionExtractor.unwrapConst(on0.child(0)); - GridSqlConst rightConst = PartitionExtractor.unwrapConst(on0.child(1)); - - if (leftConst != null && rightConst != null) { - try { - int leftConstval = leftConst.value().getInt(); - int rightConstVal = rightConst.value().getInt(); - - if (leftConstval == rightConstVal) - return PartitionJoinCondition.CROSS; - } - catch (Exception ignore) { - // No-op. - } - } - - // This is not cross-join, neither normal join between columns. - if (leftConst != null || rightConst != null) - return null; - - // Check for normal equi-join. - GridSqlColumn left = PartitionExtractor.unwrapColumn(on0.child(0)); - GridSqlColumn right = PartitionExtractor.unwrapColumn(on0.child(1)); - - if (left != null && right != null) { - String leftAlias = left.tableAlias(); - String rightAlias = right.tableAlias(); - - String leftCol = left.columnName(); - String rightCol = right.columnName(); - - return new PartitionJoinCondition(leftAlias, rightAlias, leftCol, rightCol); - } - } - } - - return null; - } - - /** - * Prepare affinity identifier for cache. - * - * @param ccfg Cache configuration. - * @return Affinity identifier. - */ - public static PartitionJoinAffinityDescriptor affinityForCache(CacheConfiguration ccfg) { - // Partition could be extracted only from PARTITIONED caches. - if (ccfg.getCacheMode() != CacheMode.PARTITIONED) - return null; - - PartitionAffinityFunctionType aff = ccfg.getAffinity().getClass().equals(RendezvousAffinityFunction.class) ? - PartitionAffinityFunctionType.RENDEZVOUS : PartitionAffinityFunctionType.CUSTOM; - - boolean hasNodeFilter = ccfg.getNodeFilter() != null && - !(ccfg.getNodeFilter() instanceof CacheConfiguration.IgniteAllNodesPredicate); - - return new PartitionJoinAffinityDescriptor( - aff, - ccfg.getAffinity().partitions(), - hasNodeFilter - ); - } - - /** - * Private constructor. - */ - private PartitionExtractorUtils() { - // No-op. - } -} diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinCondition.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionJoinCondition.java similarity index 98% rename from modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinCondition.java rename to modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionJoinCondition.java index 4f5279c47188a..244c301b951db 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinCondition.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionJoinCondition.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.internal.processors.query.h2.affinity.join; +package org.apache.ignite.internal.processors.query.h2.affinity; import org.apache.ignite.internal.util.typedef.F; diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionJoinGroup.java similarity index 74% rename from modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java rename to modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionJoinGroup.java index bf6a6ae365bc5..641d013328cdd 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinGroup.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionJoinGroup.java @@ -15,36 +15,36 @@ * limitations under the License. */ -package org.apache.ignite.internal.processors.query.h2.affinity.join; +package org.apache.ignite.internal.processors.query.h2.affinity; import java.util.Collection; import java.util.Collections; import java.util.IdentityHashMap; /** - * Partition join group. + * Group of joined tables whose affinity function could be "merged". */ @SuppressWarnings("AssignmentOrReturnOfFieldWithMutableType") public class PartitionJoinGroup { /** Tables within a group. */ - private final Collection tbls = Collections.newSetFromMap(new IdentityHashMap<>()); + private final Collection tbls = Collections.newSetFromMap(new IdentityHashMap<>()); /** Affinity function descriptor. */ - private final PartitionJoinAffinityDescriptor affDesc; + private final PartitionTableAffinityDescriptor affDesc; /** * Constructor. * * @param affDesc Affinity function descriptor. */ - public PartitionJoinGroup(PartitionJoinAffinityDescriptor affDesc) { + public PartitionJoinGroup(PartitionTableAffinityDescriptor affDesc) { this.affDesc = affDesc; } /** * @return Tables in a group. */ - public Collection tables() { + public Collection tables() { return tbls; } @@ -54,7 +54,7 @@ public Collection tables() { * @param tbl Table. * @return This for chaining. */ - public PartitionJoinGroup addTable(PartitionJoinTable tbl) { + public PartitionJoinGroup addTable(PartitionTable tbl) { tbls.add(tbl); return this; @@ -66,7 +66,7 @@ public PartitionJoinGroup addTable(PartitionJoinTable tbl) { * @param tbl Table. * @return If group is empty after removal. */ - public boolean removeTable(PartitionJoinTable tbl) { + public boolean removeTable(PartitionTable tbl) { tbls.remove(tbl); return tbls.isEmpty(); @@ -75,7 +75,7 @@ public boolean removeTable(PartitionJoinTable tbl) { /** * @return Affinity descriptor. */ - public PartitionJoinAffinityDescriptor affinityDescriptor() { + public PartitionTableAffinityDescriptor affinityDescriptor() { return affDesc; } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionNoneNode.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionNoneNode.java index e4f32811dd1ec..5d4b32433e495 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionNoneNode.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionNoneNode.java @@ -17,7 +17,6 @@ package org.apache.ignite.internal.processors.query.h2.affinity; -import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionTableModel; import org.apache.ignite.internal.util.typedef.internal.S; import java.util.Collection; diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionParameterNode.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionParameterNode.java index 40aa0c70c3429..e9f4880058c4a 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionParameterNode.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionParameterNode.java @@ -20,7 +20,6 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.internal.processors.query.h2.H2Utils; import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing; -import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinTable; import org.apache.ignite.internal.util.tostring.GridToStringExclude; import org.apache.ignite.internal.util.typedef.internal.S; @@ -46,7 +45,7 @@ public class PartitionParameterNode extends PartitionSingleNode { * @param idx Parameter index. * @param dataType Parameter data type. */ - public PartitionParameterNode(PartitionJoinTable tbl, IgniteH2Indexing indexing, int idx, + public PartitionParameterNode(PartitionTable tbl, IgniteH2Indexing indexing, int idx, int dataType) { super(tbl); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionResult.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionResult.java index a44b5311acb70..675b9b199eac7 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionResult.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionResult.java @@ -17,7 +17,6 @@ package org.apache.ignite.internal.processors.query.h2.affinity; -import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinAffinityDescriptor; import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.internal.S; @@ -30,7 +29,7 @@ public class PartitionResult { private final PartitionNode tree; /** Affinity function. */ - private final PartitionJoinAffinityDescriptor aff; + private final PartitionTableAffinityDescriptor aff; /** * Constructor. @@ -38,7 +37,7 @@ public class PartitionResult { * @param tree Tree. * @param aff Affinity function. */ - public PartitionResult(PartitionNode tree, PartitionJoinAffinityDescriptor aff) { + public PartitionResult(PartitionNode tree, PartitionTableAffinityDescriptor aff) { this.tree = tree; this.aff = aff; } @@ -53,7 +52,7 @@ public PartitionNode tree() { /** * @return Affinity function. */ - public PartitionJoinAffinityDescriptor affinity() { + public PartitionTableAffinityDescriptor affinity() { return aff; } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionSingleNode.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionSingleNode.java index d8fd7306c50ca..b9723386abc2b 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionSingleNode.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionSingleNode.java @@ -18,7 +18,6 @@ package org.apache.ignite.internal.processors.query.h2.affinity; import org.apache.ignite.IgniteCheckedException; -import org.apache.ignite.internal.processors.query.h2.affinity.join.PartitionJoinTable; import org.apache.ignite.internal.util.tostring.GridToStringExclude; import org.apache.ignite.internal.util.typedef.F; @@ -31,14 +30,14 @@ public abstract class PartitionSingleNode implements PartitionNode { /** Table descriptor. */ @GridToStringExclude - protected final PartitionJoinTable tbl; + protected final PartitionTable tbl; /** * Constructor. * * @param tbl Table descriptor. */ - protected PartitionSingleNode(PartitionJoinTable tbl) { + protected PartitionSingleNode(PartitionTable tbl) { this.tbl = tbl; } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTable.java similarity index 94% rename from modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java rename to modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTable.java index 892a1d73d8a06..609b8644c6d3c 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinTable.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTable.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.internal.processors.query.h2.affinity.join; +package org.apache.ignite.internal.processors.query.h2.affinity; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.S; @@ -24,7 +24,7 @@ /** * Single table with affinity info. */ -public class PartitionJoinTable { +public class PartitionTable { /** Alias used in the query. */ private final String alias; @@ -48,7 +48,7 @@ public class PartitionJoinTable { * @param affColName Affinity column name. * @param secondAffColName Second affinity column name. */ - public PartitionJoinTable( + public PartitionTable( String alias, String cacheName, @Nullable String affColName, @@ -128,6 +128,6 @@ public void joinGorup(int joinGrp) { /** {@inheritDoc} */ @Override public String toString() { - return S.toString(PartitionJoinTable.class, this); + return S.toString(PartitionTable.class, this); } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinAffinityDescriptor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTableAffinityDescriptor.java similarity index 88% rename from modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinAffinityDescriptor.java rename to modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTableAffinityDescriptor.java index 9b141bf9ee00d..8389f06b182e7 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionJoinAffinityDescriptor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTableAffinityDescriptor.java @@ -15,9 +15,8 @@ * limitations under the License. */ -package org.apache.ignite.internal.processors.query.h2.affinity.join; +package org.apache.ignite.internal.processors.query.h2.affinity; -import org.apache.ignite.cache.CacheMode; import org.apache.ignite.internal.util.typedef.internal.S; import java.io.Serializable; @@ -25,7 +24,7 @@ /** * Affinity function descriptor. Used to compare affinity functions of two tables. */ -public class PartitionJoinAffinityDescriptor implements Serializable { +public class PartitionTableAffinityDescriptor implements Serializable { /** */ private static final long serialVersionUID = 0L; @@ -45,7 +44,7 @@ public class PartitionJoinAffinityDescriptor implements Serializable { * @param parts Number of partitions. * @param hasNodeFilter Whether node filter is set. */ - public PartitionJoinAffinityDescriptor( + public PartitionTableAffinityDescriptor( PartitionAffinityFunctionType affFunc, int parts, boolean hasNodeFilter @@ -61,7 +60,7 @@ public PartitionJoinAffinityDescriptor( * @param other Other descriptor. * @return {@code True} if compatible. */ - public boolean isCompatible(PartitionJoinAffinityDescriptor other) { + public boolean isCompatible(PartitionTableAffinityDescriptor other) { if (other == null) return false; @@ -84,6 +83,6 @@ public boolean isCompatible(PartitionJoinAffinityDescriptor other) { /** {@inheritDoc} */ @Override public String toString() { - return S.toString(PartitionJoinAffinityDescriptor.class, this); + return S.toString(PartitionTableAffinityDescriptor.class, this); } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTableModel.java similarity index 86% rename from modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java rename to modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTableModel.java index 9fa5fdcde749f..a2d118619ad07 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/join/PartitionTableModel.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTableModel.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.internal.processors.query.h2.affinity.join; +package org.apache.ignite.internal.processors.query.h2.affinity; import org.jetbrains.annotations.Nullable; @@ -32,7 +32,7 @@ public class PartitionTableModel { public static final int GRP_NONE = -1; /** All tables observed during parsing excluding outer. */ - private final Map tbls = new HashMap<>(); + private final Map tbls = new HashMap<>(); /** Join groups. */ private final Map grps = new HashMap<>(); @@ -49,7 +49,7 @@ public class PartitionTableModel { * @param tbl Table. * @param aff Affinity descriptor. */ - public void addTable(PartitionJoinTable tbl, PartitionJoinAffinityDescriptor aff) { + public void addTable(PartitionTable tbl, PartitionTableAffinityDescriptor aff) { int grpIdx = grpIdxGen++; tbl.joinGorup(grpIdx); @@ -64,8 +64,8 @@ public void addTable(PartitionJoinTable tbl, PartitionJoinAffinityDescriptor aff * @param alias Alias. * @return Table or {@code null} if it cannot be used for partition pruning. */ - @Nullable public PartitionJoinTable table(String alias) { - PartitionJoinTable res = tbls.get(alias); + @Nullable public PartitionTable table(String alias) { + PartitionTable res = tbls.get(alias); assert res != null || (excludedTblNames != null && excludedTblNames.contains(alias)); @@ -78,7 +78,7 @@ public void addTable(PartitionJoinTable tbl, PartitionJoinAffinityDescriptor aff * @param alias Alias. */ public void addExcludedTable(String alias) { - PartitionJoinTable tbl = tbls.remove(alias); + PartitionTable tbl = tbls.remove(alias); if (tbl != null) { PartitionJoinGroup grp = grps.get(tbl.joinGroup()); @@ -101,8 +101,8 @@ public void addExcludedTable(String alias) { * @param cond Condition. */ public void addJoin(PartitionJoinCondition cond) { - PartitionJoinTable leftTbl = tbls.get(cond.leftAlias()); - PartitionJoinTable rightTbl = tbls.get(cond.rightAlias()); + PartitionTable leftTbl = tbls.get(cond.leftAlias()); + PartitionTable rightTbl = tbls.get(cond.rightAlias()); assert leftTbl != null || (excludedTblNames != null && excludedTblNames.contains(cond.leftAlias())); assert rightTbl != null || (excludedTblNames != null && excludedTblNames.contains(cond.rightAlias())); @@ -129,7 +129,7 @@ public void addJoin(PartitionJoinCondition cond) { return; // Safe to merge groups. - for (PartitionJoinTable tbl : rightGrp.tables()) { + for (PartitionTable tbl : rightGrp.tables()) { tbl.joinGorup(leftTbl.joinGroup()); leftGrp.addTable(tbl); @@ -144,7 +144,7 @@ public void addJoin(PartitionJoinCondition cond) { * @param grpId Group ID. * @return Affinity descriptor or {@code null} if there is no affinity descriptor (e.g. for "NONE" result). */ - @Nullable public PartitionJoinAffinityDescriptor joinGroupAffinity(int grpId) { + @Nullable public PartitionTableAffinityDescriptor joinGroupAffinity(int grpId) { if (grpId == GRP_NONE) return null; diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index 59c7c4b28f02c..e611c6ebe6c21 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -353,6 +353,14 @@ public void testOuterJoin() { // TODO } + /** + * Make sure that partition ownership is not merged when tables are "joined" under OR condition. + */ + @Test + public void testJoinUnderDisjunction() { + // TODO + } + /** * Create PARTITIONED table. * From 0b5b0a6a92e67532cb796c43d1d5719c7fe8feca Mon Sep 17 00:00:00 2001 From: devozerov Date: Sat, 29 Dec 2018 13:01:19 +0300 Subject: [PATCH 46/80] WIP. --- .../twostep/JoinPartitionPruningSelfTest.java | 45 +++++++++++++------ 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index e611c6ebe6c21..a9bf2dc8fe97e 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -207,34 +207,49 @@ public void testSimpleJoin() { } /** - * Test various expressions. + * Test how partition ownership is transferred in various cases. */ @Test - public void testExpressions() { - // TODO - } - - /** - * Test simple join. - */ - @Test - public void testComplexJoin() { - // TODO + public void testPartitionTransfer() { + // First co-located table. createPartitionedTable("t1", pkColumn("k1"), "v2"); + // Second co-located table. createPartitionedTable("t2", pkColumn("k1"), affinityColumn("ak2"), "v3"); - execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.ak2 = 1 OR t1.k1 = 2"); + // Replicated table. + createReplicatedTable("t3", + pkColumn("k1"), + "v2", + "v3" + ); + // Transfer through "AND". + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = 1 AND t2.ak2 = 1"); assertPartitions( - parititon("t2", "1"), - parititon("t1", "2") + parititon("t1", "1") ); + + // TODO: AND (empty) + + // Transfer through "OR". + // TODO + + // No transfer through intermediate table. + // TODO + } + + /** + * Test various expressions. + */ + @Test + public void testExpressions() { + // TODO } /** @@ -448,6 +463,8 @@ private void createTable0(String name, boolean replicated, Object... cols) { * @param sql SQL. */ private List> execute(String sql, Object... args) { + clearIoState(); + SqlFieldsQuery qry = new SqlFieldsQuery(sql); if (args != null && args.length > 0) From 774bb3a864f93f722beca19603d5a1358eff409e Mon Sep 17 00:00:00 2001 From: devozerov Date: Sat, 29 Dec 2018 13:13:54 +0300 Subject: [PATCH 47/80] WIP. --- .../h2/affinity/PartitionCompositeNode.java | 2 +- .../h2/affinity/PartitionSingleNode.java | 4 +- .../twostep/JoinPartitionPruningSelfTest.java | 64 ++++++++----------- 3 files changed, 30 insertions(+), 40 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionCompositeNode.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionCompositeNode.java index fe43b0b2b0594..5d130b6eb2132 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionCompositeNode.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionCompositeNode.java @@ -301,7 +301,7 @@ private PartitionNode optimizeSimpleAnd(PartitionSingleNode left, PartitionSingl // Currently we do not merge such nodes because it may violate existing broken (!!!) join semantics. // Normally, if we have two non-collocated partition sets, then this should be an empty set for collocated - // query mode. Unfortunately, // current semantics of collocated query mode assume that even though both sides + // query mode. Unfortunately, current semantics of collocated query mode assume that even though both sides // of expression are located on random nodes, there is a slight chance that they may accidentally reside on // a single node and hence return some rows. We return "ALL" here to keep this broken semantics consistent // irrespective of whether partition pruning is used or not. Once non-collocated joins are fixed, this diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionSingleNode.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionSingleNode.java index b9723386abc2b..f210c98b73700 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionSingleNode.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionSingleNode.java @@ -74,7 +74,7 @@ protected PartitionSingleNode(PartitionTable tbl) { int hash = (constant() ? 1 : 0); hash = 31 * hash + value(); - hash = 31 * hash + tbl.alias().hashCode(); + hash = 31 * hash + tbl.joinGroup(); return hash; } @@ -90,6 +90,6 @@ protected PartitionSingleNode(PartitionTable tbl) { PartitionSingleNode other = (PartitionSingleNode)obj; return F.eq(constant(), other.constant()) && F.eq(value(), other.value()) && - F.eq(tbl.alias(), other.tbl.alias()); + F.eq(tbl.joinGroup(), other.tbl.joinGroup()); } } diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index a9bf2dc8fe97e..833002d49ba59 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -127,28 +127,28 @@ public void testSimpleJoin() { // Key (no alias). List> res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = 1"); - assertPartitionsAndClear( + assertPartitions( parititon("t1", "1") ); assertEquals(1, res.size()); assertEquals("1", res.get(0).get(0)); res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = '1'"); - assertPartitionsAndClear( + assertPartitions( parititon("t1", "1") ); assertEquals(1, res.size()); assertEquals("1", res.get(0).get(0)); res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ?", 1); - assertPartitionsAndClear( + assertPartitions( parititon("t1", "1") ); assertEquals(1, res.size()); assertEquals("1", res.get(0).get(0)); res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ?", "1"); - assertPartitionsAndClear( + assertPartitions( parititon("t1", "1") ); assertEquals(1, res.size()); @@ -156,14 +156,14 @@ public void testSimpleJoin() { // Key (alias). res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1._KEY = '2'"); - assertPartitionsAndClear( + assertPartitions( parititon("t1", "2") ); assertEquals(1, res.size()); assertEquals("2", res.get(0).get(0)); res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1._KEY = ?", "2"); - assertPartitionsAndClear( + assertPartitions( parititon("t1", "2") ); assertEquals(1, res.size()); @@ -182,14 +182,14 @@ public void testSimpleJoin() { // Affinity key. res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.ak2 = 4"); - assertPartitionsAndClear( + assertPartitions( parititon("t2", "4") ); assertEquals(1, res.size()); assertEquals("4", res.get(0).get(0)); res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.ak2 = ?", "4"); - assertPartitionsAndClear( + assertPartitions( parititon("t2", "4") ); assertEquals(1, res.size()); @@ -199,7 +199,7 @@ public void testSimpleJoin() { BinaryObject key = client().binary().builder("t2_key").setField("k1", "5").setField("ak2", "5").build(); res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2._KEY = ?", key); - assertPartitionsAndClear( + assertPartitions( parititon("t2", "5") ); assertEquals(1, res.size()); @@ -214,16 +214,25 @@ public void testPartitionTransfer() { // First co-located table. createPartitionedTable("t1", pkColumn("k1"), - "v2"); + "v2" + ); // Second co-located table. createPartitionedTable("t2", pkColumn("k1"), affinityColumn("ak2"), - "v3"); + "v3" + ); + + // Third co-located table. + createPartitionedTable("t3", + pkColumn("k1"), + affinityColumn("ak2"), + "v3" + ); // Replicated table. - createReplicatedTable("t3", + createReplicatedTable("t4", pkColumn("k1"), "v2", "v3" @@ -235,7 +244,8 @@ public void testPartitionTransfer() { parititon("t1", "1") ); - // TODO: AND (empty) + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = 1 AND t2.ak2 = 2"); + assertNoRequests(); // Transfer through "OR". // TODO @@ -277,14 +287,14 @@ public void testCrossJoin() { // Left table, should work. List> res = execute("SELECT * FROM t1, t2 WHERE t1.k1 = '1'"); - assertPartitionsAndClear( + assertPartitions( parititon("t1", "1") ); assertEquals(1, res.size()); assertEquals("1", res.get(0).get(0)); res = execute("SELECT * FROM t1 INNER JOIN t2 ON 1=1 WHERE t1.k1 = '1'"); - assertPartitionsAndClear( + assertPartitions( parititon("t1", "1") ); assertEquals(1, res.size()); @@ -292,14 +302,14 @@ public void testCrossJoin() { // Right table, should work. res = execute("SELECT * FROM t1, t2 WHERE t2.ak2 = '2'"); - assertPartitionsAndClear( + assertPartitions( parititon("t2", "2") ); assertEquals(1, res.size()); assertEquals("2", res.get(0).get(0)); res = execute("SELECT * FROM t1 INNER JOIN t2 ON 1=1 WHERE t2.ak2 = '2'"); - assertPartitionsAndClear( + assertPartitions( parititon("t2", "2") ); assertEquals(1, res.size()); @@ -488,17 +498,6 @@ private static void clearIoState() { INTERCEPTED_PARTS.clear(); } - /** - * Make sure that expected partitions are logged, then clear IO state. - * - * @param expParts Expected partitions. - */ - private static void assertPartitionsAndClear(int... expParts) { - assertPartitions(expParts); - - clearIoState(); - } - /** * Make sure that expected partitions are logged. * @@ -526,15 +525,6 @@ private static void assertPartitions(Collection expParts) { expParts0, actualParts); } - /** - * Make sure that no partitions were extracted, then clear IO state. - */ - private static void assertNoPartitionsAndClear() { - assertNoPartitions(); - - clearIoState(); - } - /** * Make sure that no partitions were extracted. */ From c6250956c2992c4843759b60af9b7a66ecadcaf8 Mon Sep 17 00:00:00 2001 From: devozerov Date: Sat, 29 Dec 2018 13:42:57 +0300 Subject: [PATCH 48/80] Fixed composite nodes. --- .../h2/affinity/PartitionCompositeNode.java | 59 +++++++++++++++---- .../h2/affinity/PartitionSingleNode.java | 11 +++- .../twostep/JoinPartitionPruningSelfTest.java | 6 ++ 3 files changed, 61 insertions(+), 15 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionCompositeNode.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionCompositeNode.java index 5d130b6eb2132..2300f0c8afc64 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionCompositeNode.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionCompositeNode.java @@ -19,6 +19,7 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.internal.util.tostring.GridToStringInclude; +import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.S; import java.util.Collection; @@ -65,6 +66,8 @@ public PartitionCompositeNode(PartitionNode left, PartitionNode right, Partition return null; // (A, B) and (B, C) -> (B) + leftParts = new HashSet<>(leftParts); + leftParts.retainAll(rightParts); } else { @@ -77,6 +80,8 @@ else if (rightParts == null) return leftParts; // (A, B) or (B, C) -> (A, B, C) + leftParts = new HashSet<>(leftParts); + leftParts.addAll(rightParts); } @@ -223,18 +228,46 @@ else if (right instanceof PartitionGroupNode) { } if (rightConsts != null) { - // {A, B) and (B, C) -> (B). - consts.retainAll(rightConsts); - - if (consts.isEmpty()) - // {A, B) and (C, D) -> NONE. - return PartitionNoneNode.INSTANCE; - else if (consts.size() == 1) + // Try to merge nodes if the belong to the same table. + boolean sameTbl = true; + String curTblAlias = null; + + for (PartitionSingleNode curConst : consts) { + if (curTblAlias == null) + curTblAlias = curConst.table().alias(); + else if (!F.eq(curTblAlias, curConst.table().alias())) { + sameTbl = false; + + break; + } + } + + if (sameTbl) { + for (PartitionSingleNode curConst : rightConsts) { + if (curTblAlias == null) + curTblAlias = curConst.table().alias(); + else if (!F.eq(curTblAlias, curConst.table().alias())) { + sameTbl = false; + + break; + } + } + } + + if (sameTbl) { // {A, B) and (B, C) -> (B). - return consts.iterator().next(); - else - // {A, B, C) and (B, C, D) -> (B, C). - return new PartitionGroupNode(consts); + consts.retainAll(rightConsts); + + if (consts.isEmpty()) + // {A, B) and (C, D) -> NONE. + return PartitionNoneNode.INSTANCE; + else if (consts.size() == 1) + // {A, B) and (B, C) -> (B). + return consts.iterator().next(); + else + // {A, B, C) and (B, C, D) -> (B, C). + return new PartitionGroupNode(consts); + } } } @@ -315,8 +348,8 @@ private PartitionNode optimizeSimpleAnd(PartitionSingleNode left, PartitionSingl // (:X) and (:X) -> :X return left; - // If both sides are constants, and they are not equal, this is empty set. - if (left.constant() && right.constant()) + // If both sides are constants from the same table and they are not equal, this is empty set. + if (left.constant() && right.constant() && F.eq(left.table().alias(), right.tbl.alias())) // X and Y -> NONE return PartitionNoneNode.INSTANCE; diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionSingleNode.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionSingleNode.java index f210c98b73700..35e7d30fef475 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionSingleNode.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionSingleNode.java @@ -69,12 +69,19 @@ protected PartitionSingleNode(PartitionTable tbl) { */ public abstract int value(); + /** + * @return Underlying table. + */ + public PartitionTable table() { + return tbl; + } + /** {@inheritDoc} */ @Override public int hashCode() { int hash = (constant() ? 1 : 0); hash = 31 * hash + value(); - hash = 31 * hash + tbl.joinGroup(); + hash = 31 * hash + tbl.alias().hashCode(); return hash; } @@ -90,6 +97,6 @@ protected PartitionSingleNode(PartitionTable tbl) { PartitionSingleNode other = (PartitionSingleNode)obj; return F.eq(constant(), other.constant()) && F.eq(value(), other.value()) && - F.eq(tbl.joinGroup(), other.tbl.joinGroup()); + F.eq(tbl.alias(), other.tbl.alias()); } } diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index 833002d49ba59..2b859326ff5cd 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -247,6 +247,12 @@ public void testPartitionTransfer() { execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = 1 AND t2.ak2 = 2"); assertNoRequests(); + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 IN (1, 2) AND t2.ak2 IN (2, 3)"); + assertPartitions( + parititon("t1", "2") + ); + + // Transfer through "OR". // TODO From 2eee6b6da47b3e51bcbe7f3f34951a1651e335a0 Mon Sep 17 00:00:00 2001 From: devozerov Date: Sat, 29 Dec 2018 13:48:19 +0300 Subject: [PATCH 49/80] WIP. --- .../h2/twostep/JoinPartitionPruningSelfTest.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index 2b859326ff5cd..4f8d1dd76400f 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -247,11 +247,22 @@ public void testPartitionTransfer() { execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = 1 AND t2.ak2 = 2"); assertNoRequests(); + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = 1 AND t2.ak2 IN (1, 2)"); + assertPartitions( + parititon("t1", "1") + ); + + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = 1 AND t2.ak2 IN (2, 3)"); + assertNoRequests(); + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 IN (1, 2) AND t2.ak2 IN (2, 3)"); assertPartitions( parititon("t1", "2") ); + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 IN (1, 2) AND t2.ak2 IN (3, 4)"); + assertNoRequests(); + // Transfer through "OR". // TODO From 63ecc10b40ba7f5bf947f426878f5cac737afea2 Mon Sep 17 00:00:00 2001 From: devozerov Date: Sat, 29 Dec 2018 14:20:12 +0300 Subject: [PATCH 50/80] WIP on combinatorial transfer. --- .../twostep/JoinPartitionPruningSelfTest.java | 135 +++++++++++++++++- 1 file changed, 132 insertions(+), 3 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index 4f8d1dd76400f..c16c22ed1a4b6 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -38,11 +38,13 @@ import org.junit.runners.JUnit4; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.TreeSet; import java.util.concurrent.ConcurrentSkipListSet; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; /** * Tests for join partition pruning. @@ -239,21 +241,73 @@ public void testPartitionTransfer() { ); // Transfer through "AND". - execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = 1 AND t2.ak2 = 1"); - assertPartitions( - parititon("t1", "1") + executeCombinations("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 = ?", + (res) -> assertPartitions( + parititon("t1", "1") + ), + "1", "1" ); + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = 1 AND t2.ak2 = 2"); assertNoRequests(); + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 = 2", "1"); + assertNoRequests(); + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 = ?", "1", "2"); + assertNoRequests(); execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = 1 AND t2.ak2 IN (1, 2)"); assertPartitions( parititon("t1", "1") ); + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 IN (1, 2)", "1"); + assertPartitions( + parititon("t1", "1") + ); + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = 1 AND t2.ak2 IN (?, 2)", "1"); + assertPartitions( + parititon("t1", "1") + ); + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 IN (?, 2)", "1", "1"); + assertPartitions( + parititon("t1", "1") + ); + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = 1 AND t2.ak2 IN (?, ?)", "1", "2"); + assertPartitions( + parititon("t1", "1") + ); + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 IN (?, ?)", "1", "1", "2"); + assertPartitions( + parititon("t1", "1") + ); execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = 1 AND t2.ak2 IN (2, 3)"); assertNoRequests(); + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 IN (2, 3)", "1"); + assertNoRequests(); + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = 1 AND t2.ak2 IN (?, 3)", "2"); + assertNoRequests(); + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = 1 AND t2.ak2 IN (2, ?)", "3"); + assertNoRequests(); + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 IN (?, 3)", "1", "2"); + assertNoRequests(); + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 IN (2, ?)", "1", "3"); + assertNoRequests(); + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = 1 AND t2.ak2 IN (?, ?)", "2", "3"); + assertNoRequests(); + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 IN (?, ?)", "1", "2", "3"); + assertNoRequests(); + + + + + + + + + + + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 IN (1, 2) AND t2.ak2 IN (2, 3)"); assertPartitions( @@ -484,6 +538,76 @@ private void createTable0(String name, boolean replicated, Object... cols) { execute(sql.toString()); } + /** + * Execute query with all possible combinations of argument placeholders. + * + * @param sql SQL. + * @param resConsumer Result consumer. + * @param args Arguments. + */ + public void executeCombinations(String sql, Consumer>> resConsumer, Object... args) { + // Execute query as is. + List> res = execute(sql, args); + + resConsumer.accept(res); + + // Start filling arguments recursively. + if (args != null && args.length > 0) + executeCombinations0(sql, resConsumer, args); + } + + /** + * Execute query with all possible combinations of argument placeholders. + * + * @param sql SQL. + * @param resConsumer Result consumer. + * @param args Arguments. + */ + public void executeCombinations0(String sql, Consumer>> resConsumer, Object... args) { + assert args != null && args.length > 0; + + // Get argument positions. + List paramPoss = new ArrayList<>(); + + int pos = 0; + + while (true) { + int paramPos = sql.indexOf('?', pos); + + if (paramPos == -1) + break; + + paramPoss.add(paramPos); + + pos = paramPos + 1; + } + + for (int i = 0; i < args.length; i++) { + // Prepare new SQL and arguments. + int paramPos = paramPoss.get(i); + + String newSql = sql.substring(0, paramPos) + args[i] + sql.substring(paramPos + 1); + + Object[] newArgs = new Object[args.length - 1]; + + int newArgsPos = 0; + + for (int j = 0; j < args.length; j++) { + if (j != i) + newArgs[newArgsPos++] = args[j]; + } + + // Execute. + List> res = execute(newSql, newArgs); + + resConsumer.accept(res); + + // Continue recursively. + if (newArgs.length > 0) + executeCombinations0(newSql, resConsumer, newArgs); + } + } + /** * Execute SQL query. * @@ -492,6 +616,11 @@ private void createTable0(String name, boolean replicated, Object... cols) { private List> execute(String sql, Object... args) { clearIoState(); + if (args == null || args.length == 0) + System.out.println(">>> " + sql); + else + System.out.println(">>> " + sql + " " + Arrays.toString(args)); + SqlFieldsQuery qry = new SqlFieldsQuery(sql); if (args != null && args.length > 0) From 7cfa7edd5525a48a6ba0d1cc189312a03a3daae7 Mon Sep 17 00:00:00 2001 From: devozerov Date: Sat, 29 Dec 2018 14:35:30 +0300 Subject: [PATCH 51/80] WIP on combinatorial transfer. --- .../twostep/JoinPartitionPruningSelfTest.java | 129 ++++++++---------- 1 file changed, 60 insertions(+), 69 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index c16c22ed1a4b6..3445ec9f9e8e1 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -40,6 +40,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.TreeSet; import java.util.concurrent.ConcurrentSkipListSet; @@ -248,75 +249,34 @@ public void testPartitionTransfer() { "1", "1" ); - - execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = 1 AND t2.ak2 = 2"); - assertNoRequests(); - execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 = 2", "1"); - assertNoRequests(); - execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 = ?", "1", "2"); - assertNoRequests(); - - execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = 1 AND t2.ak2 IN (1, 2)"); - assertPartitions( - parititon("t1", "1") - ); - execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 IN (1, 2)", "1"); - assertPartitions( - parititon("t1", "1") - ); - execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = 1 AND t2.ak2 IN (?, 2)", "1"); - assertPartitions( - parititon("t1", "1") - ); - execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 IN (?, 2)", "1", "1"); - assertPartitions( - parititon("t1", "1") - ); - execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = 1 AND t2.ak2 IN (?, ?)", "1", "2"); - assertPartitions( - parititon("t1", "1") - ); - execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 IN (?, ?)", "1", "1", "2"); - assertPartitions( - parititon("t1", "1") + executeCombinations("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 = ?", + (res) -> assertNoRequests(), + "1", "2" ); - execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = 1 AND t2.ak2 IN (2, 3)"); - assertNoRequests(); - execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 IN (2, 3)", "1"); - assertNoRequests(); - execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = 1 AND t2.ak2 IN (?, 3)", "2"); - assertNoRequests(); - execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = 1 AND t2.ak2 IN (2, ?)", "3"); - assertNoRequests(); - execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 IN (?, 3)", "1", "2"); - assertNoRequests(); - execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 IN (2, ?)", "1", "3"); - assertNoRequests(); - execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = 1 AND t2.ak2 IN (?, ?)", "2", "3"); - assertNoRequests(); - execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 IN (?, ?)", "1", "2", "3"); - assertNoRequests(); - - - - - - - - - - - + executeCombinations("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 IN (?, ?)", + (res) -> assertPartitions( + parititon("t1", "1") + ), + "1", "1", "2" + ); - execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 IN (1, 2) AND t2.ak2 IN (2, 3)"); - assertPartitions( - parititon("t1", "2") + executeCombinations("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 IN (?, ?)", + (res) -> assertNoRequests(), + "1", "2", "3" ); - execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 IN (1, 2) AND t2.ak2 IN (3, 4)"); - assertNoRequests(); + executeCombinations("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 IN (?, ?) AND t2.ak2 IN (?, ?)", + (res) -> assertPartitions( + parititon("t1", "2") + ), + "1", "2", "2", "3" + ); + executeCombinations("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 IN (?, ?) AND t2.ak2 IN (?, ?)", + (res) -> assertNoRequests(), + "1", "2", "3", "4" + ); // Transfer through "OR". // TODO @@ -546,6 +506,8 @@ private void createTable0(String name, boolean replicated, Object... cols) { * @param args Arguments. */ public void executeCombinations(String sql, Consumer>> resConsumer, Object... args) { + System.out.println(">>> TEST COMBINATION: " + sql); + // Execute query as is. List> res = execute(sql, args); @@ -553,7 +515,9 @@ public void executeCombinations(String sql, Consumer>> resConsumer, // Start filling arguments recursively. if (args != null && args.length > 0) - executeCombinations0(sql, resConsumer, args); + executeCombinations0(sql, resConsumer, new TreeSet<>(), new HashSet<>(), args); + + System.out.println(); } /** @@ -561,9 +525,17 @@ public void executeCombinations(String sql, Consumer>> resConsumer, * * @param sql SQL. * @param resConsumer Result consumer. + * @param excludedArgs Arguments which are currently excluded. + * @param testedExcludedArgs Combinations of excluded arguments which already were tested. * @param args Arguments. */ - public void executeCombinations0(String sql, Consumer>> resConsumer, Object... args) { + public void executeCombinations0( + String sql, + Consumer>> resConsumer, + TreeSet excludedArgs, + HashSet testedExcludedArgs, + Object... args + ) { assert args != null && args.length > 0; // Get argument positions. @@ -597,14 +569,33 @@ public void executeCombinations0(String sql, Consumer>> resConsumer newArgs[newArgsPos++] = args[j]; } - // Execute. - List> res = execute(newSql, newArgs); + // Execute if this combination was never executed before. + excludedArgs.add(i); + + StringBuilder combinationKey = new StringBuilder(); + + boolean first = true; + + for (Integer excludedArg : excludedArgs) { + if (first) + first = false; + else + combinationKey.append("-"); + + combinationKey.append(excludedArg); + } + + if (testedExcludedArgs.add(combinationKey.toString())) { + List> res = execute(newSql, newArgs); - resConsumer.accept(res); + resConsumer.accept(res); + } // Continue recursively. if (newArgs.length > 0) - executeCombinations0(newSql, resConsumer, newArgs); + executeCombinations0(newSql, resConsumer, excludedArgs, testedExcludedArgs, newArgs); + + excludedArgs.remove(i); } } From 7ebd83b85127bcd0dcb874e562dd5eef4b78f398 Mon Sep 17 00:00:00 2001 From: devozerov Date: Sat, 29 Dec 2018 14:42:35 +0300 Subject: [PATCH 52/80] Testing. --- .../twostep/JoinPartitionPruningSelfTest.java | 73 +++++++++++++------ 1 file changed, 49 insertions(+), 24 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index 3445ec9f9e8e1..0f2b423dc4774 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -317,41 +317,66 @@ public void testCrossJoin() { execute("INSERT INTO t2 VALUES ('3', '3', '3')"); // Left table, should work. - List> res = execute("SELECT * FROM t1, t2 WHERE t1.k1 = '1'"); - assertPartitions( - parititon("t1", "1") + executeCombinations("SELECT * FROM t1, t2 WHERE t1.k1 = ?", + (res) -> { + assertPartitions( + parititon("t1", "1") + ); + assertEquals(1, res.size()); + assertEquals("1", res.get(0).get(0)); + }, + "1" ); - assertEquals(1, res.size()); - assertEquals("1", res.get(0).get(0)); - res = execute("SELECT * FROM t1 INNER JOIN t2 ON 1=1 WHERE t1.k1 = '1'"); - assertPartitions( - parititon("t1", "1") + executeCombinations("SELECT * FROM t1 INNER JOIN t2 ON 1=1 WHERE t1.k1 = ?", + (res) -> { + assertPartitions( + parititon("t1", "1") + ); + assertEquals(1, res.size()); + assertEquals("1", res.get(0).get(0)); + }, + "1" ); - assertEquals(1, res.size()); - assertEquals("1", res.get(0).get(0)); // Right table, should work. - res = execute("SELECT * FROM t1, t2 WHERE t2.ak2 = '2'"); - assertPartitions( - parititon("t2", "2") + executeCombinations("SELECT * FROM t1, t2 WHERE t2.ak2 = ?", + (res) -> { + assertPartitions( + parititon("t2", "2") + ); + assertEquals(1, res.size()); + assertEquals("2", res.get(0).get(0)); + }, + "2" ); - assertEquals(1, res.size()); - assertEquals("2", res.get(0).get(0)); - res = execute("SELECT * FROM t1 INNER JOIN t2 ON 1=1 WHERE t2.ak2 = '2'"); - assertPartitions( - parititon("t2", "2") + executeCombinations("SELECT * FROM t1 INNER JOIN t2 ON 1=1 WHERE t2.ak2 = ?", + (res) -> { + assertPartitions( + parititon("t2", "2") + ); + assertEquals(1, res.size()); + assertEquals("2", res.get(0).get(0)); + }, + "2" + ); + + executeCombinations("SELECT * FROM t1, t2 WHERE t1.k1=? AND t2.ak2 = ?", + (res) -> assertNoPartitions(), + "3", "3" ); - assertEquals(1, res.size()); - assertEquals("2", res.get(0).get(0)); // Two tables, should not work. - res = execute("SELECT * FROM t1, t2 WHERE t1.k1='3' AND t2.ak2 = '3'"); - assertNoPartitions(); + executeCombinations("SELECT * FROM t1, t2 WHERE t1.k1=? AND t2.ak2 = ?", + (res) -> assertNoPartitions(), + "3", "3" + ); - res = execute("SELECT * FROM t1, t2 WHERE t1.k1='3' OR t2.ak2 = '3'"); - assertNoPartitions(); + executeCombinations("SELECT * FROM t1 INNER JOIN t2 ON 1=1 WHERE t1.k1=? AND t2.ak2 = ?", + (res) -> assertNoPartitions(), + "3", "3" + ); } /** From d802b628bb21980bb7b69954ed1cc95262e5e2f4 Mon Sep 17 00:00:00 2001 From: devozerov Date: Sat, 29 Dec 2018 14:52:14 +0300 Subject: [PATCH 53/80] WIP. --- .../twostep/JoinPartitionPruningSelfTest.java | 110 +++++++----------- 1 file changed, 41 insertions(+), 69 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index 0f2b423dc4774..64fef8bb1a761 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -50,6 +50,7 @@ /** * Tests for join partition pruning. */ +@SuppressWarnings("deprecation") @RunWith(JUnit4.class) public class JoinPartitionPruningSelfTest extends GridCommonAbstractTest { /** Number of intercepted requests. */ @@ -128,80 +129,56 @@ public void testSimpleJoin() { execute("INSERT INTO t1 VALUES ('5', '5')"); execute("INSERT INTO t2 VALUES ('5', '5', '5')"); - // Key (no alias). - List> res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = 1"); - assertPartitions( - parititon("t1", "1") - ); - assertEquals(1, res.size()); - assertEquals("1", res.get(0).get(0)); - - res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = '1'"); - assertPartitions( - parititon("t1", "1") - ); - assertEquals(1, res.size()); - assertEquals("1", res.get(0).get(0)); - - res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ?", 1); - assertPartitions( - parititon("t1", "1") - ); - assertEquals(1, res.size()); - assertEquals("1", res.get(0).get(0)); - - res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ?", "1"); - assertPartitions( - parititon("t1", "1") + // Key (not alias). + executeCombinations("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ?", + (res) -> { + assertPartitions( + parititon("t1", "1") + ); + assertEquals(1, res.size()); + assertEquals("1", res.get(0).get(0)); + }, + "1" ); - assertEquals(1, res.size()); - assertEquals("1", res.get(0).get(0)); // Key (alias). - res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1._KEY = '2'"); - assertPartitions( - parititon("t1", "2") - ); - assertEquals(1, res.size()); - assertEquals("2", res.get(0).get(0)); - - res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1._KEY = ?", "2"); - assertPartitions( - parititon("t1", "2") + executeCombinations("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1._KEY = ?", + (res) -> { + assertPartitions( + parititon("t1", "2") + ); + assertEquals(1, res.size()); + assertEquals("2", res.get(0).get(0)); + }, + "2" ); - assertEquals(1, res.size()); - assertEquals("2", res.get(0).get(0)); // Non-affinity key. - res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.k1 = 3"); - assertNoPartitions(); - assertEquals(1, res.size()); - assertEquals("3", res.get(0).get(0)); - - res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.k1 = ?", "3"); - assertNoPartitions(); - assertEquals(1, res.size()); - assertEquals("3", res.get(0).get(0)); - - // Affinity key. - res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.ak2 = 4"); - assertPartitions( - parititon("t2", "4") + executeCombinations("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.k1 = ?", + (res) -> { + assertNoPartitions(); + assertEquals(1, res.size()); + assertEquals("3", res.get(0).get(0)); + }, + "3" ); - assertEquals(1, res.size()); - assertEquals("4", res.get(0).get(0)); - res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.ak2 = ?", "4"); - assertPartitions( - parititon("t2", "4") + // Affinity key. + executeCombinations("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.ak2 = ?", + (res) -> { + assertPartitions( + parititon("t2", "4") + ); + assertEquals(1, res.size()); + assertEquals("4", res.get(0).get(0)); + }, + "4" ); - assertEquals(1, res.size()); - assertEquals("4", res.get(0).get(0)); // Complex key. BinaryObject key = client().binary().builder("t2_key").setField("k1", "5").setField("ak2", "5").build(); - res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2._KEY = ?", key); + List> res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2._KEY = ?", key); assertPartitions( parititon("t2", "5") ); @@ -283,6 +260,9 @@ public void testPartitionTransfer() { // No transfer through intermediate table. // TODO + + // No transfer through disjunction. + // TODO } /** @@ -434,14 +414,6 @@ public void testOuterJoin() { // TODO } - /** - * Make sure that partition ownership is not merged when tables are "joined" under OR condition. - */ - @Test - public void testJoinUnderDisjunction() { - // TODO - } - /** * Create PARTITIONED table. * From 1f9af776767f4c65bca7526c254e64695737df0b Mon Sep 17 00:00:00 2001 From: devozerov Date: Sat, 29 Dec 2018 14:58:25 +0300 Subject: [PATCH 54/80] Fixed combinatorial stuff. --- .../twostep/JoinPartitionPruningSelfTest.java | 31 +++++-------------- 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index 64fef8bb1a761..a65168c054c85 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -42,6 +42,7 @@ import java.util.Collection; import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.TreeSet; import java.util.concurrent.ConcurrentSkipListSet; import java.util.concurrent.atomic.AtomicInteger; @@ -512,7 +513,7 @@ public void executeCombinations(String sql, Consumer>> resConsumer, // Start filling arguments recursively. if (args != null && args.length > 0) - executeCombinations0(sql, resConsumer, new TreeSet<>(), new HashSet<>(), args); + executeCombinations0(sql, resConsumer, new HashSet<>(), args); System.out.println(); } @@ -522,15 +523,14 @@ public void executeCombinations(String sql, Consumer>> resConsumer, * * @param sql SQL. * @param resConsumer Result consumer. - * @param excludedArgs Arguments which are currently excluded. - * @param testedExcludedArgs Combinations of excluded arguments which already were tested. + * @param executedSqls Already executed SQLs. * @param args Arguments. */ + // TODO: Fix combinatorial checks! public void executeCombinations0( String sql, Consumer>> resConsumer, - TreeSet excludedArgs, - HashSet testedExcludedArgs, + Set executedSqls, Object... args ) { assert args != null && args.length > 0; @@ -567,22 +567,7 @@ public void executeCombinations0( } // Execute if this combination was never executed before. - excludedArgs.add(i); - - StringBuilder combinationKey = new StringBuilder(); - - boolean first = true; - - for (Integer excludedArg : excludedArgs) { - if (first) - first = false; - else - combinationKey.append("-"); - - combinationKey.append(excludedArg); - } - - if (testedExcludedArgs.add(combinationKey.toString())) { + if (executedSqls.add(newSql)) { List> res = execute(newSql, newArgs); resConsumer.accept(res); @@ -590,9 +575,7 @@ public void executeCombinations0( // Continue recursively. if (newArgs.length > 0) - executeCombinations0(newSql, resConsumer, excludedArgs, testedExcludedArgs, newArgs); - - excludedArgs.remove(i); + executeCombinations0(newSql, resConsumer, executedSqls, newArgs); } } From 63c54c948c050cc77629a2c1dc40fba5c637af92 Mon Sep 17 00:00:00 2001 From: devozerov Date: Sat, 29 Dec 2018 15:06:59 +0300 Subject: [PATCH 55/80] WIP. --- .../twostep/JoinPartitionPruningSelfTest.java | 79 +++++++++---------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index a65168c054c85..dfede56276809 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -115,23 +115,23 @@ public void testSimpleJoin() { affinityColumn("ak2"), "v3"); - execute("INSERT INTO t1 VALUES ('1', '1')"); - execute("INSERT INTO t2 VALUES ('1', '1', '1')"); + executeSingle("INSERT INTO t1 VALUES ('1', '1')"); + executeSingle("INSERT INTO t2 VALUES ('1', '1', '1')"); - execute("INSERT INTO t1 VALUES ('2', '2')"); - execute("INSERT INTO t2 VALUES ('2', '2', '2')"); + executeSingle("INSERT INTO t1 VALUES ('2', '2')"); + executeSingle("INSERT INTO t2 VALUES ('2', '2', '2')"); - execute("INSERT INTO t1 VALUES ('3', '3')"); - execute("INSERT INTO t2 VALUES ('3', '3', '3')"); + executeSingle("INSERT INTO t1 VALUES ('3', '3')"); + executeSingle("INSERT INTO t2 VALUES ('3', '3', '3')"); - execute("INSERT INTO t1 VALUES ('4', '4')"); - execute("INSERT INTO t2 VALUES ('4', '4', '4')"); + executeSingle("INSERT INTO t1 VALUES ('4', '4')"); + executeSingle("INSERT INTO t2 VALUES ('4', '4', '4')"); - execute("INSERT INTO t1 VALUES ('5', '5')"); - execute("INSERT INTO t2 VALUES ('5', '5', '5')"); + executeSingle("INSERT INTO t1 VALUES ('5', '5')"); + executeSingle("INSERT INTO t2 VALUES ('5', '5', '5')"); // Key (not alias). - executeCombinations("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ?", + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ?", (res) -> { assertPartitions( parititon("t1", "1") @@ -143,7 +143,7 @@ public void testSimpleJoin() { ); // Key (alias). - executeCombinations("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1._KEY = ?", + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1._KEY = ?", (res) -> { assertPartitions( parititon("t1", "2") @@ -155,7 +155,7 @@ public void testSimpleJoin() { ); // Non-affinity key. - executeCombinations("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.k1 = ?", + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.k1 = ?", (res) -> { assertNoPartitions(); assertEquals(1, res.size()); @@ -165,7 +165,7 @@ public void testSimpleJoin() { ); // Affinity key. - executeCombinations("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.ak2 = ?", + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.ak2 = ?", (res) -> { assertPartitions( parititon("t2", "4") @@ -179,7 +179,7 @@ public void testSimpleJoin() { // Complex key. BinaryObject key = client().binary().builder("t2_key").setField("k1", "5").setField("ak2", "5").build(); - List> res = execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2._KEY = ?", key); + List> res = executeSingle("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2._KEY = ?", key); assertPartitions( parititon("t2", "5") ); @@ -220,38 +220,38 @@ public void testPartitionTransfer() { ); // Transfer through "AND". - executeCombinations("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 = ?", + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 = ?", (res) -> assertPartitions( parititon("t1", "1") ), "1", "1" ); - executeCombinations("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 = ?", + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 = ?", (res) -> assertNoRequests(), "1", "2" ); - executeCombinations("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 IN (?, ?)", + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 IN (?, ?)", (res) -> assertPartitions( parititon("t1", "1") ), "1", "1", "2" ); - executeCombinations("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 IN (?, ?)", + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 IN (?, ?)", (res) -> assertNoRequests(), "1", "2", "3" ); - executeCombinations("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 IN (?, ?) AND t2.ak2 IN (?, ?)", + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 IN (?, ?) AND t2.ak2 IN (?, ?)", (res) -> assertPartitions( parititon("t1", "2") ), "1", "2", "2", "3" ); - executeCombinations("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 IN (?, ?) AND t2.ak2 IN (?, ?)", + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 IN (?, ?) AND t2.ak2 IN (?, ?)", (res) -> assertNoRequests(), "1", "2", "3", "4" ); @@ -288,17 +288,17 @@ public void testCrossJoin() { affinityColumn("ak2"), "v3"); - execute("INSERT INTO t1 VALUES ('1', '1')"); - execute("INSERT INTO t2 VALUES ('1', '1', '1')"); + executeSingle("INSERT INTO t1 VALUES ('1', '1')"); + executeSingle("INSERT INTO t2 VALUES ('1', '1', '1')"); - execute("INSERT INTO t1 VALUES ('2', '2')"); - execute("INSERT INTO t2 VALUES ('2', '2', '2')"); + executeSingle("INSERT INTO t1 VALUES ('2', '2')"); + executeSingle("INSERT INTO t2 VALUES ('2', '2', '2')"); - execute("INSERT INTO t1 VALUES ('3', '3')"); - execute("INSERT INTO t2 VALUES ('3', '3', '3')"); + executeSingle("INSERT INTO t1 VALUES ('3', '3')"); + executeSingle("INSERT INTO t2 VALUES ('3', '3', '3')"); // Left table, should work. - executeCombinations("SELECT * FROM t1, t2 WHERE t1.k1 = ?", + execute("SELECT * FROM t1, t2 WHERE t1.k1 = ?", (res) -> { assertPartitions( parititon("t1", "1") @@ -309,7 +309,7 @@ public void testCrossJoin() { "1" ); - executeCombinations("SELECT * FROM t1 INNER JOIN t2 ON 1=1 WHERE t1.k1 = ?", + execute("SELECT * FROM t1 INNER JOIN t2 ON 1=1 WHERE t1.k1 = ?", (res) -> { assertPartitions( parititon("t1", "1") @@ -321,7 +321,7 @@ public void testCrossJoin() { ); // Right table, should work. - executeCombinations("SELECT * FROM t1, t2 WHERE t2.ak2 = ?", + execute("SELECT * FROM t1, t2 WHERE t2.ak2 = ?", (res) -> { assertPartitions( parititon("t2", "2") @@ -332,7 +332,7 @@ public void testCrossJoin() { "2" ); - executeCombinations("SELECT * FROM t1 INNER JOIN t2 ON 1=1 WHERE t2.ak2 = ?", + execute("SELECT * FROM t1 INNER JOIN t2 ON 1=1 WHERE t2.ak2 = ?", (res) -> { assertPartitions( parititon("t2", "2") @@ -343,18 +343,18 @@ public void testCrossJoin() { "2" ); - executeCombinations("SELECT * FROM t1, t2 WHERE t1.k1=? AND t2.ak2 = ?", + execute("SELECT * FROM t1, t2 WHERE t1.k1=? AND t2.ak2 = ?", (res) -> assertNoPartitions(), "3", "3" ); // Two tables, should not work. - executeCombinations("SELECT * FROM t1, t2 WHERE t1.k1=? AND t2.ak2 = ?", + execute("SELECT * FROM t1, t2 WHERE t1.k1=? AND t2.ak2 = ?", (res) -> assertNoPartitions(), "3", "3" ); - executeCombinations("SELECT * FROM t1 INNER JOIN t2 ON 1=1 WHERE t1.k1=? AND t2.ak2 = ?", + execute("SELECT * FROM t1 INNER JOIN t2 ON 1=1 WHERE t1.k1=? AND t2.ak2 = ?", (res) -> assertNoPartitions(), "3", "3" ); @@ -493,7 +493,7 @@ private void createTable0(String name, boolean replicated, Object... cols) { sql.append("\""); - execute(sql.toString()); + executeSingle(sql.toString()); } /** @@ -503,11 +503,11 @@ private void createTable0(String name, boolean replicated, Object... cols) { * @param resConsumer Result consumer. * @param args Arguments. */ - public void executeCombinations(String sql, Consumer>> resConsumer, Object... args) { + public void execute(String sql, Consumer>> resConsumer, Object... args) { System.out.println(">>> TEST COMBINATION: " + sql); // Execute query as is. - List> res = execute(sql, args); + List> res = executeSingle(sql, args); resConsumer.accept(res); @@ -526,7 +526,6 @@ public void executeCombinations(String sql, Consumer>> resConsumer, * @param executedSqls Already executed SQLs. * @param args Arguments. */ - // TODO: Fix combinatorial checks! public void executeCombinations0( String sql, Consumer>> resConsumer, @@ -568,7 +567,7 @@ public void executeCombinations0( // Execute if this combination was never executed before. if (executedSqls.add(newSql)) { - List> res = execute(newSql, newArgs); + List> res = executeSingle(newSql, newArgs); resConsumer.accept(res); } @@ -584,7 +583,7 @@ public void executeCombinations0( * * @param sql SQL. */ - private List> execute(String sql, Object... args) { + private List> executeSingle(String sql, Object... args) { clearIoState(); if (args == null || args.length == 0) From e291837d8d8b9068190be5773e3a093c21aefc83 Mon Sep 17 00:00:00 2001 From: devozerov Date: Sat, 29 Dec 2018 16:01:22 +0300 Subject: [PATCH 56/80] Minors. --- .../processors/query/h2/IgniteH2Indexing.java | 1 - .../h2/affinity/PartitionCompositeNode.java | 4 +- .../query/h2/affinity/PartitionExtractor.java | 5 +- .../query/h2/affinity/PartitionTable.java | 14 ---- .../query/QueryRewritePlaygroundTest.java | 70 ------------------- 5 files changed, 5 insertions(+), 89 deletions(-) delete mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/QueryRewritePlaygroundTest.java diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index be58f7ebda735..fe014c2202005 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -1999,7 +1999,6 @@ private FieldsQueryCursor> doRunDistributedQuery(String schemaName, SqlF return new QueryCursorImpl<>(new Iterable>() { @Override public Iterator> iterator() { return new Iterator>() { - @Override public boolean hasNext() { return false; } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionCompositeNode.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionCompositeNode.java index 2300f0c8afc64..45ceaaff368e8 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionCompositeNode.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionCompositeNode.java @@ -228,7 +228,7 @@ else if (right instanceof PartitionGroupNode) { } if (rightConsts != null) { - // Try to merge nodes if the belong to the same table. + // Try to merge nodes if they belong to the same table. boolean sameTbl = true; String curTblAlias = null; @@ -271,7 +271,7 @@ else if (consts.size() == 1) } } - // Otherwise it is a mixed set of concrete partitions and arguments. Cancel optimization. + // Otherwise it is a mixed set of concrete partitions and arguments possibly from different caches. // Note that in fact we can optimize expression to certain extent (e.g. (A) and (B, :C) -> (A) and (:C)), // but resulting expression is always composite node still, which cannot be optimized on upper levels. // So we skip any fine-grained optimization in favor of simplicity. diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java index cfeb1a5810ac1..822107fd230ed 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java @@ -179,7 +179,7 @@ private List prepareTableModel0(GridSqlAst from, PartitionTableM if (join.isLeftOuter()) { // "a LEFT JOIN b" is transformed into "a", and "b" is put into special stop-list. - // If a condition is met on "b" afterwards, we will stop partition pruning process. + // If a condition is met on "b" afterwards, we will ignore it. for (PartitionTable rightTbl : rightTbls) model.addExcludedTable(rightTbl.alias()); @@ -305,6 +305,7 @@ private static PartitionTable prepareTable(GridSqlAst from, PartitionTableModel } PartitionTable tbl = new PartitionTable(alias, cacheName, affColName, secondAffColName); + PartitionTableAffinityDescriptor aff = affinityForCache(tbl0.cacheInfo().config()); if (aff == null) { @@ -360,7 +361,7 @@ private static PartitionTableAffinityDescriptor affinityForCache(CacheConfigurat * @param disjunct Whether current processing frame is located under disjunction ("OR"). In this case we cannot * rely on join expressions like (A.a = B.b) to build co-location model because another conflicting * join expression on the same tables migth be located on the other side of the "OR". - * Example: "JOIN on A.a = B.b OR A.a > B.b". + * Example: "JOIN ON A.a = B.b OR A.a > B.b". * @return Partition tree. */ @SuppressWarnings("EnumSwitchStatementWhichMissesCases") diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTable.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTable.java index 609b8644c6d3c..8b14b7805a314 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTable.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTable.java @@ -88,20 +88,6 @@ public boolean hasAffinityColumn() { return affColName != null; } - /** - * @return Affinity column name. - */ - public String affinityColumnName() { - return affColName; - } - - /** - * @return Second affinity column name. - */ - public String secondAffinityColumnName() { - return secondAffColName; - } - /** * Check whether passed column is affinity column. * diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/QueryRewritePlaygroundTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/QueryRewritePlaygroundTest.java deleted file mode 100644 index 55dd601e92451..0000000000000 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/QueryRewritePlaygroundTest.java +++ /dev/null @@ -1,70 +0,0 @@ -package org.apache.ignite.internal.processors.query; - -import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; -import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; -import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.Statement; - -public class QueryRewritePlaygroundTest extends GridCommonAbstractTest { - /** IP finder. */ - private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); - - /** {@inheritDoc} */ - @Override protected void beforeTest() throws Exception { - startGrid(); - } - - /** {@inheritDoc} */ - @Override protected void afterTest() throws Exception { - stopAllGrids(); - } - - /** {@inheritDoc} */ - @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - - cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER)); - - return cfg; - } - - /** {@inheritDoc} */ - @Override protected long getTestTimeout() { - return Long.MAX_VALUE; - } - - /** - * @throws Exception If failed. - */ - public void testVarious() throws Exception { - executeSql("CREATE TABLE dept (id BIGINT PRIMARY KEY, name VARCHAR)"); - executeSql("CREATE TABLE emp (id BIGINT, name VARCHAR, dept_id BIGINT, salary BIGINT, PRIMARY KEY (id, dept_id)) WITH \"affinity_key=dept_id\""); - - //executeSql("SELECT emp.name, dept.name FROM emp, dept WHERE emp.dept_id=dept.id"); -// executeSql("SELECT emp.name, dept.name FROM emp INNER JOIN dept ON emp.dept_id=dept.id"); - -// executeSql("" + -// "SELECT emp.name, (SELECT dept.name FROM dept WHERE emp.dept_id=dept.id)\n" + -// "FROM emp\n" + -// "WHERE emp.salary > 1000" -// ); - - executeSql("" + - "SELECT * FROM emp e " + - " LEFT JOIN dept d ON e.dept_id = d.id " + - " LEFT JOIN dept d2 ON e.dept_id > d2.id" - ); - } - - public static void executeSql(String sql) throws Exception { - try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1")) { - try (Statement stmt = conn.createStatement()) { - stmt.execute(sql); - } - } - } -} From 6e3fd5c15c0b3c3740926ee8c74e70c3d197f8a7 Mon Sep 17 00:00:00 2001 From: devozerov Date: Sat, 29 Dec 2018 16:01:54 +0300 Subject: [PATCH 57/80] Minors. --- .../InOperationExtractPartitionSelfTest.java | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/InOperationExtractPartitionSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/InOperationExtractPartitionSelfTest.java index 969420c6d8de9..c3222de3ae7bf 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/InOperationExtractPartitionSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/InOperationExtractPartitionSelfTest.java @@ -156,6 +156,30 @@ public void testSingleValueList() { @Test public void testMultipleValueList() { testInOperator(Arrays.asList(ORG + 0, ORG + 3, ORG + String.valueOf(ORG_COUNT - 1)), null, 3, 3); + testInOperator(Arrays.asList("ORG", ORG + 0, ORG + 4, ORG + String.valueOf(ORG_COUNT - 1)), null, 3, 4); + testInOperator(Arrays.asList(ORG + 0, ORG + 5, ORG + String.valueOf(ORG_COUNT - 1), "ORG"), null, 3, 4); + testInOperator(Arrays.asList(ORG + 0, ORG + 6, "MID", ORG + String.valueOf(ORG_COUNT - 1), "ORG"), null, 3, 5); + + final List allArgs3 = Arrays.asList("?", "?", "?"); + final List allArgs4 = Arrays.asList("?", "?", "?", "?"); + + testInOperator(allArgs3, new String[] {ORG + 0, ORG + 7, ORG + String.valueOf(ORG_COUNT - 1)}, 3, 3); + testInOperator(allArgs4, new String[] {"ORG", ORG + 0, ORG + 8, ORG + String.valueOf(ORG_COUNT - 1)}, 3, 4); + testInOperator(allArgs4, new String[] {ORG + 0, ORG + 9, ORG + String.valueOf(ORG_COUNT - 1), "ORG"}, 3, 4); + testInOperator(allArgs4, new String[] {ORG + 0, "MID", ORG + String.valueOf(ORG_COUNT - 1), "ORG"}, 2, 4); + + testInOperator( + Arrays.asList("?", ORG + 9, ORG + String.valueOf(ORG_COUNT - 1), "?"), + new String[] {ORG + 0, "ORG"}, + 3, + 4 + ); + testInOperator( + Arrays.asList("?", "?", ORG + String.valueOf(ORG_COUNT - 1), "ORG"), + new String[] {ORG + 0, "MID"}, + 2, + 4 + ); } /** From a14ed2c0cb6c76dce996c14501b661c43650afed Mon Sep 17 00:00:00 2001 From: devozerov Date: Sat, 29 Dec 2018 16:02:01 +0300 Subject: [PATCH 58/80] Minors. --- .../query/h2/twostep/InOperationExtractPartitionSelfTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/InOperationExtractPartitionSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/InOperationExtractPartitionSelfTest.java index c3222de3ae7bf..926232fd1bbda 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/InOperationExtractPartitionSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/InOperationExtractPartitionSelfTest.java @@ -24,7 +24,6 @@ import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteException; import org.apache.ignite.cache.CacheMode; -import org.apache.ignite.cache.QueryEntity; import org.apache.ignite.cache.query.FieldsQueryCursor; import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.cluster.ClusterNode; From 456a7ec5166b6ff2ab7aa877622692ff6fa2a998 Mon Sep 17 00:00:00 2001 From: devozerov Date: Sat, 29 Dec 2018 16:07:30 +0300 Subject: [PATCH 59/80] Tests: or transfer. --- .../twostep/JoinPartitionPruningSelfTest.java | 53 ++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index dfede56276809..aeb34eb65b8f2 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -257,7 +257,58 @@ public void testPartitionTransfer() { ); // Transfer through "OR". - // TODO + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? OR t2.ak2 = ?", + (res) -> assertPartitions( + parititon("t1", "1") + ), + "1", "1" + ); + + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? OR t2.ak2 = ?", + (res) -> assertPartitions( + parititon("t1", "1"), + parititon("t2", "2") + ), + "1", "2" + ); + + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? OR t2.ak2 IN (?, ?)", + (res) -> assertPartitions( + parititon("t1", "1"), + parititon("t2", "2") + ), + "1", "1", "2" + ); + + + + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? OR t2.ak2 IN (?, ?)", + (res) -> assertPartitions( + parititon("t1", "1"), + parititon("t2", "2"), + parititon("t2", "3") + ), + "1", "2", "3" + ); + + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 IN (?, ?) OR t2.ak2 IN (?, ?)", + (res) -> assertPartitions( + parititon("t1", "1"), + parititon("t1", "2"), + parititon("t2", "3") + ), + "1", "2", "2", "3" + ); + + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 IN (?, ?) OR t2.ak2 IN (?, ?)", + (res) -> assertPartitions( + parititon("t1", "1"), + parititon("t1", "2"), + parititon("t2", "3"), + parititon("t2", "4") + ), + "1", "2", "3", "4" + ); // No transfer through intermediate table. // TODO From 573634ea31682331c226e9cc77b130ff4e16ba2e Mon Sep 17 00:00:00 2001 From: devozerov Date: Sat, 29 Dec 2018 16:08:22 +0300 Subject: [PATCH 60/80] Minors. --- .../query/h2/twostep/JoinPartitionPruningSelfTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index aeb34eb65b8f2..4a2a3cda3fc3d 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -280,8 +280,6 @@ public void testPartitionTransfer() { "1", "1", "2" ); - - execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? OR t2.ak2 IN (?, ?)", (res) -> assertPartitions( parititon("t1", "1"), From f3b3f2d34c597a6c6b6c27d8b5dac3accd6d7e0a Mon Sep 17 00:00:00 2001 From: devozerov Date: Wed, 16 Jan 2019 12:01:20 +0300 Subject: [PATCH 61/80] More tests. --- .../twostep/JoinPartitionPruningSelfTest.java | 29 +++++++++++++++++-- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index 4a2a3cda3fc3d..9848c4d4f6a8a 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -209,7 +209,8 @@ public void testPartitionTransfer() { createPartitionedTable("t3", pkColumn("k1"), affinityColumn("ak2"), - "v3" + "v3", + "v4" ); // Replicated table. @@ -308,11 +309,33 @@ public void testPartitionTransfer() { "1", "2", "3", "4" ); + // Multi-way co-located JOIN. + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 INNER JOIN t3 ON t1.k1 = t3.ak2 " + + "WHERE t1.k1 = ? AND t2.ak2 = ? AND t3.ak2 = ?", + (res) -> assertPartitions( + parititon("t1", "1") + ), + "1", "1", "1" + ); + + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 INNER JOIN t3 ON t1.k1 = t3.ak2 " + + "WHERE t1.k1 = ? AND t2.ak2 = ? AND t3.ak2 = ?", + (res) -> assertNoRequests(), + "1", "2", "3" + ); + // No transfer through intermediate table. - // TODO + execute("SELECT * FROM t1 INNER JOIN t3 ON t1.k1 = t3.v3 INNER JOIN t2 ON t3.v4 = t2.ak2 " + + "WHERE t1.k1 = ? AND t2.ak2 = ?", + (res) -> assertNoPartitions(), + "1", "1" + ); // No transfer through disjunction. - // TODO + execute("SELECT * FROM t1 INNER JOIN t2 ON 1=1 WHERE t1.k1 = ? OR t1.k1 = t2.ak2", + (res) -> assertNoPartitions(), + "1" + ); } /** From 8fa118dae189e36cc6afacb7920703e529f2bd1e Mon Sep 17 00:00:00 2001 From: devozerov Date: Wed, 16 Jan 2019 12:04:11 +0300 Subject: [PATCH 62/80] WIP on tests. --- .../query/h2/twostep/JoinPartitionPruningSelfTest.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index 9848c4d4f6a8a..ca9dc867a9d7f 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -338,14 +338,6 @@ public void testPartitionTransfer() { ); } - /** - * Test various expressions. - */ - @Test - public void testExpressions() { - // TODO - } - /** * Test cross-joins. They cannot "transfer" partitions between joined tables. */ From e23c7a36e564111006da4edd9c6ca2f7ddceaa02 Mon Sep 17 00:00:00 2001 From: devozerov Date: Wed, 16 Jan 2019 12:10:29 +0300 Subject: [PATCH 63/80] WIP on tests. --- .../twostep/JoinPartitionPruningSelfTest.java | 87 ++++++++++++++++++- 1 file changed, 85 insertions(+), 2 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index ca9dc867a9d7f..e87b1dc9fc5d0 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -213,6 +213,7 @@ public void testPartitionTransfer() { "v4" ); + // TODO: Move to RPELICATED table tests. // Replicated table. createReplicatedTable("t4", pkColumn("k1"), @@ -429,11 +430,93 @@ public void testCrossJoin() { */ @Test public void testThetaJoin() { - // TODO + createPartitionedTable("t1", + pkColumn("k1"), + "v2"); + + createPartitionedTable("t2", + pkColumn("k1"), + affinityColumn("ak2"), + "v3"); + + // Greater than. + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 > t2.ak2 WHERE t1.k1 = ?", + (res) -> assertPartitions( + parititon("t1", "1") + ), + "1" + ); + + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 > t2.ak2 WHERE t2.ak2 = ?", + (res) -> assertPartitions( + parititon("t1", "1") + ), + "1" + ); + + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 > t2.ak2 WHERE t1.k1 = ? AND t2.ak2 = ?", + (res) -> assertNoPartitions(), + "1", "1" + ); + + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 > t2.ak2 WHERE t1.k1 = ? OR t2.ak2 = ?", + (res) -> assertNoPartitions(), + "1", "2" + ); + + // Less than. + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 < t2.ak2 WHERE t1.k1 = ?", + (res) -> assertPartitions( + parititon("t1", "1") + ), + "1" + ); + + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 < t2.ak2 WHERE t2.ak2 = ?", + (res) -> assertPartitions( + parititon("t1", "1") + ), + "1" + ); + + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 < t2.ak2 WHERE t1.k1 = ? AND t2.ak2 = ?", + (res) -> assertNoPartitions(), + "1", "1" + ); + + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 < t2.ak2 WHERE t1.k1 = ? OR t2.ak2 = ?", + (res) -> assertNoPartitions(), + "1", "2" + ); + + // Non-equal. + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 <> t2.ak2 WHERE t1.k1 = ?", + (res) -> assertPartitions( + parititon("t1", "1") + ), + "1" + ); + + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 <> t2.ak2 WHERE t2.ak2 = ?", + (res) -> assertPartitions( + parititon("t1", "1") + ), + "1" + ); + + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 <> t2.ak2 WHERE t1.k1 = ? AND t2.ak2 = ?", + (res) -> assertNoPartitions(), + "1", "1" + ); + + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 <> t2.ak2 WHERE t1.k1 = ? OR t2.ak2 = ?", + (res) -> assertNoPartitions(), + "1", "2" + ); } /** - * Test joins with REPLICTED caches. + * Test joins with REPLICATED caches. */ @Test public void testJoinWithReplicated() { From c17c8d60c018c295a9adb6c201abaf3eb69b63da Mon Sep 17 00:00:00 2001 From: devozerov Date: Wed, 16 Jan 2019 13:03:03 +0300 Subject: [PATCH 64/80] Tests for REPLICATED cache. --- .../twostep/JoinPartitionPruningSelfTest.java | 49 +++++++++++++++---- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index e87b1dc9fc5d0..8d6729cd39c40 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -213,14 +213,6 @@ public void testPartitionTransfer() { "v4" ); - // TODO: Move to RPELICATED table tests. - // Replicated table. - createReplicatedTable("t4", - pkColumn("k1"), - "v2", - "v3" - ); - // Transfer through "AND". execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 = ?", (res) -> assertPartitions( @@ -516,11 +508,48 @@ public void testThetaJoin() { } /** - * Test joins with REPLICATED caches. + * Test joins with REPLICATED cache. */ @Test public void testJoinWithReplicated() { - // TODO + // First co-located table. + createPartitionedTable("t1", + pkColumn("k1"), + "v2" + ); + + // Replicated table. + createReplicatedTable("t2", + pkColumn("k1"), + "v2", + "v3" + ); + + // Only partition from PARTITIONED cache should be used. + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.k1 WHERE t1.k1 = ? AND t2.k1 = ?", + (res) -> assertPartitions( + parititon("t1", "1") + ), + "1", "2" + ); + + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.k1 WHERE t1.k1 IN (?, ?) AND t2.k1 = ?", + (res) -> assertPartitions( + parititon("t1", "1"), + parititon("t1", "2") + ), + "1", "2", "3" + ); + + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.k1 WHERE t1.k1 = ? OR t2.k1 = ?", + (res) -> assertNoPartitions(), + "1", "2" + ); + + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.k1 WHERE t2.k1 = ?", + (res) -> assertNoPartitions(), + "1" + ); } /** From 1ac62ad0ec1b29fc766270b98007bdb755b3cba5 Mon Sep 17 00:00:00 2001 From: devozerov Date: Wed, 16 Jan 2019 16:27:18 +0300 Subject: [PATCH 65/80] Self-join tests. --- .../twostep/JoinPartitionPruningSelfTest.java | 50 ++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index 8d6729cd39c40..5dec6e9a63350 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -579,7 +579,7 @@ public void testJoinWithSubquery() { * Test joins when explicit partitions are set. */ @Test - public void testExplicitParititons() { + public void testExplicitPartitions() { // TODO } @@ -588,9 +588,57 @@ public void testExplicitParititons() { */ @Test public void testOuterJoin() { + createPartitionedTable("t1", + pkColumn("k1"), + "v2"); + + createPartitionedTable("t2", + pkColumn("k1"), + affinityColumn("ak2"), + "v3"); + + + // TODO } + /** + * Test JOINs on a single table. + */ + @Test + public void testSelfJoin() { + createPartitionedTable("t1", + pkColumn("k1"), + "v2"); + + execute("SELECT * FROM t1 A INNER JOIN t1 B ON A.k1 = B.k1 WHERE A.k1 = ?", + (res) -> assertPartitions( + parititon("t1", "1") + ), + "1" + ); + + execute("SELECT * FROM t1 A INNER JOIN t1 B ON A.k1 = B.k1 WHERE A.k1 = ? AND B.k1 = ?", + (res) -> assertPartitions( + parititon("t1", "1") + ), + "1", "1" + ); + + execute("SELECT * FROM t1 A INNER JOIN t1 B ON A.k1 = B.k1 WHERE A.k1 = ? AND B.k1 = ?", + (res) -> assertNoRequests(), + "1", "2" + ); + + execute("SELECT * FROM t1 A INNER JOIN t1 B ON A.k1 = B.k1 WHERE A.k1 = ? OR B.k1 = ?", + (res) -> assertPartitions( + parititon("t1", "1"), + parititon("t1", "2") + ), + "1", "2" + ); + } + /** * Create PARTITIONED table. * From 8a131a5eb8ae3abe5f49f7ef49a75d002631e62f Mon Sep 17 00:00:00 2001 From: devozerov Date: Wed, 16 Jan 2019 16:33:01 +0300 Subject: [PATCH 66/80] Outer join tests. --- .../twostep/JoinPartitionPruningSelfTest.java | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index 5dec6e9a63350..4fa309cace95f 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -597,9 +597,33 @@ public void testOuterJoin() { affinityColumn("ak2"), "v3"); + execute("SELECT * FROM t1 LEFT OUTER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ?", + (res) -> assertPartitions( + parititon("t1", "1") + ), + "1" + ); + execute("SELECT * FROM t1 LEFT OUTER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.ak2 = ?", + (res) -> assertNoPartitions(), + "1" + ); - // TODO + execute("SELECT * FROM t1 LEFT OUTER JOIN t2 T2_1 ON t1.k1 = T2_1.ak2 INNER JOIN t2 T2_2 ON T2_1.k1 = T2_2.k1 " + + "WHERE T2_2.ak2 = ?", + (res) -> assertPartitions( + parititon("t2", "1") + ), + "1" + ); + + execute("SELECT * FROM t1 LEFT OUTER JOIN t2 T2_1 ON t1.k1 = T2_1.ak2 INNER JOIN t2 T2_2 ON t1.k1 = T2_2.ak2 " + + "WHERE T2_1.ak2 = ? AND T2_2.ak2=?", + (res) -> assertPartitions( + parititon("t2", "2") + ), + "1", "2" + ); } /** From 41b6321a0141943d10a266b8f284ba18d9c7aebe Mon Sep 17 00:00:00 2001 From: devozerov Date: Wed, 16 Jan 2019 16:36:13 +0300 Subject: [PATCH 67/80] Tests for subqueries. --- .../twostep/JoinPartitionPruningSelfTest.java | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index 4fa309cace95f..25dd6411a5def 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -572,7 +572,26 @@ public void testJoinWithDifferentAffinityFunctions() { */ @Test public void testJoinWithSubquery() { - // TODO + createPartitionedTable("t1", + pkColumn("k1"), + "v2"); + + createPartitionedTable("t2", + pkColumn("k1"), + affinityColumn("ak2"), + "v3"); + + execute("SELECT * FROM t1 INNER JOIN (SELECT * FROM t2) T2_SUB ON t1.k1 = T2_SUB.ak2 WHERE t1.k1 = ?", + (res) -> assertPartitions( + parititon("t1", "1") + ), + "1" + ); + + execute("SELECT * FROM t1 INNER JOIN (SELECT * FROM t2) T2_SUB ON t1.k1 = T2_SUB.ak2 WHERE T2_SUB.ak2 = ?", + (res) -> assertNoPartitions(), + "1" + ); } /** From 693686e6f3001aad8ca6bd19d584b211f127dc13 Mon Sep 17 00:00:00 2001 From: devozerov Date: Wed, 16 Jan 2019 17:06:32 +0300 Subject: [PATCH 68/80] More tests. --- .../twostep/JoinPartitionPruningSelfTest.java | 120 +++++++++++------- 1 file changed, 74 insertions(+), 46 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index 25dd6411a5def..43465cbe93b36 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -134,7 +134,7 @@ public void testSimpleJoin() { execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ?", (res) -> { assertPartitions( - parititon("t1", "1") + partition("t1", "1") ); assertEquals(1, res.size()); assertEquals("1", res.get(0).get(0)); @@ -146,7 +146,7 @@ public void testSimpleJoin() { execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1._KEY = ?", (res) -> { assertPartitions( - parititon("t1", "2") + partition("t1", "2") ); assertEquals(1, res.size()); assertEquals("2", res.get(0).get(0)); @@ -168,7 +168,7 @@ public void testSimpleJoin() { execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.ak2 = ?", (res) -> { assertPartitions( - parititon("t2", "4") + partition("t2", "4") ); assertEquals(1, res.size()); assertEquals("4", res.get(0).get(0)); @@ -181,7 +181,7 @@ public void testSimpleJoin() { List> res = executeSingle("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2._KEY = ?", key); assertPartitions( - parititon("t2", "5") + partition("t2", "5") ); assertEquals(1, res.size()); assertEquals("5", res.get(0).get(0)); @@ -216,7 +216,7 @@ public void testPartitionTransfer() { // Transfer through "AND". execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 = ?", (res) -> assertPartitions( - parititon("t1", "1") + partition("t1", "1") ), "1", "1" ); @@ -228,7 +228,7 @@ public void testPartitionTransfer() { execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? AND t2.ak2 IN (?, ?)", (res) -> assertPartitions( - parititon("t1", "1") + partition("t1", "1") ), "1", "1", "2" ); @@ -240,7 +240,7 @@ public void testPartitionTransfer() { execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 IN (?, ?) AND t2.ak2 IN (?, ?)", (res) -> assertPartitions( - parititon("t1", "2") + partition("t1", "2") ), "1", "2", "2", "3" ); @@ -253,51 +253,51 @@ public void testPartitionTransfer() { // Transfer through "OR". execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? OR t2.ak2 = ?", (res) -> assertPartitions( - parititon("t1", "1") + partition("t1", "1") ), "1", "1" ); execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? OR t2.ak2 = ?", (res) -> assertPartitions( - parititon("t1", "1"), - parititon("t2", "2") + partition("t1", "1"), + partition("t2", "2") ), "1", "2" ); execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? OR t2.ak2 IN (?, ?)", (res) -> assertPartitions( - parititon("t1", "1"), - parititon("t2", "2") + partition("t1", "1"), + partition("t2", "2") ), "1", "1", "2" ); execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? OR t2.ak2 IN (?, ?)", (res) -> assertPartitions( - parititon("t1", "1"), - parititon("t2", "2"), - parititon("t2", "3") + partition("t1", "1"), + partition("t2", "2"), + partition("t2", "3") ), "1", "2", "3" ); execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 IN (?, ?) OR t2.ak2 IN (?, ?)", (res) -> assertPartitions( - parititon("t1", "1"), - parititon("t1", "2"), - parititon("t2", "3") + partition("t1", "1"), + partition("t1", "2"), + partition("t2", "3") ), "1", "2", "2", "3" ); execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 IN (?, ?) OR t2.ak2 IN (?, ?)", (res) -> assertPartitions( - parititon("t1", "1"), - parititon("t1", "2"), - parititon("t2", "3"), - parititon("t2", "4") + partition("t1", "1"), + partition("t1", "2"), + partition("t2", "3"), + partition("t2", "4") ), "1", "2", "3", "4" ); @@ -306,7 +306,7 @@ public void testPartitionTransfer() { execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 INNER JOIN t3 ON t1.k1 = t3.ak2 " + "WHERE t1.k1 = ? AND t2.ak2 = ? AND t3.ak2 = ?", (res) -> assertPartitions( - parititon("t1", "1") + partition("t1", "1") ), "1", "1", "1" ); @@ -358,7 +358,7 @@ public void testCrossJoin() { execute("SELECT * FROM t1, t2 WHERE t1.k1 = ?", (res) -> { assertPartitions( - parititon("t1", "1") + partition("t1", "1") ); assertEquals(1, res.size()); assertEquals("1", res.get(0).get(0)); @@ -369,7 +369,7 @@ public void testCrossJoin() { execute("SELECT * FROM t1 INNER JOIN t2 ON 1=1 WHERE t1.k1 = ?", (res) -> { assertPartitions( - parititon("t1", "1") + partition("t1", "1") ); assertEquals(1, res.size()); assertEquals("1", res.get(0).get(0)); @@ -381,7 +381,7 @@ public void testCrossJoin() { execute("SELECT * FROM t1, t2 WHERE t2.ak2 = ?", (res) -> { assertPartitions( - parititon("t2", "2") + partition("t2", "2") ); assertEquals(1, res.size()); assertEquals("2", res.get(0).get(0)); @@ -392,7 +392,7 @@ public void testCrossJoin() { execute("SELECT * FROM t1 INNER JOIN t2 ON 1=1 WHERE t2.ak2 = ?", (res) -> { assertPartitions( - parititon("t2", "2") + partition("t2", "2") ); assertEquals(1, res.size()); assertEquals("2", res.get(0).get(0)); @@ -434,14 +434,14 @@ public void testThetaJoin() { // Greater than. execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 > t2.ak2 WHERE t1.k1 = ?", (res) -> assertPartitions( - parititon("t1", "1") + partition("t1", "1") ), "1" ); execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 > t2.ak2 WHERE t2.ak2 = ?", (res) -> assertPartitions( - parititon("t1", "1") + partition("t1", "1") ), "1" ); @@ -459,14 +459,14 @@ public void testThetaJoin() { // Less than. execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 < t2.ak2 WHERE t1.k1 = ?", (res) -> assertPartitions( - parititon("t1", "1") + partition("t1", "1") ), "1" ); execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 < t2.ak2 WHERE t2.ak2 = ?", (res) -> assertPartitions( - parititon("t1", "1") + partition("t1", "1") ), "1" ); @@ -484,14 +484,14 @@ public void testThetaJoin() { // Non-equal. execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 <> t2.ak2 WHERE t1.k1 = ?", (res) -> assertPartitions( - parititon("t1", "1") + partition("t1", "1") ), "1" ); execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 <> t2.ak2 WHERE t2.ak2 = ?", (res) -> assertPartitions( - parititon("t1", "1") + partition("t1", "1") ), "1" ); @@ -528,15 +528,15 @@ public void testJoinWithReplicated() { // Only partition from PARTITIONED cache should be used. execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.k1 WHERE t1.k1 = ? AND t2.k1 = ?", (res) -> assertPartitions( - parititon("t1", "1") + partition("t1", "1") ), "1", "2" ); execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.k1 WHERE t1.k1 IN (?, ?) AND t2.k1 = ?", (res) -> assertPartitions( - parititon("t1", "1"), - parititon("t1", "2") + partition("t1", "1"), + partition("t1", "2") ), "1", "2", "3" ); @@ -565,6 +565,12 @@ public void testJoinWithDifferentAffinityFunctions() { // Node filters. // TODO + + // Backups. + // TODO + + // With and without persistence. + // TODO } /** @@ -583,7 +589,7 @@ public void testJoinWithSubquery() { execute("SELECT * FROM t1 INNER JOIN (SELECT * FROM t2) T2_SUB ON t1.k1 = T2_SUB.ak2 WHERE t1.k1 = ?", (res) -> assertPartitions( - parititon("t1", "1") + partition("t1", "1") ), "1" ); @@ -599,7 +605,19 @@ public void testJoinWithSubquery() { */ @Test public void testExplicitPartitions() { - // TODO + createPartitionedTable("t1", + pkColumn("k1"), + "v2"); + + createPartitionedTable("t2", + pkColumn("k1"), + affinityColumn("ak2"), + "v3"); + + executeSqlFieldsQuery(new SqlFieldsQuery("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 " + + "WHERE t1.k1=? OR t2.ak2=?").setArgs("1", "2").setPartitions(1)); + + assertPartitions(1); } /** @@ -618,7 +636,7 @@ public void testOuterJoin() { execute("SELECT * FROM t1 LEFT OUTER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ?", (res) -> assertPartitions( - parititon("t1", "1") + partition("t1", "1") ), "1" ); @@ -631,7 +649,7 @@ public void testOuterJoin() { execute("SELECT * FROM t1 LEFT OUTER JOIN t2 T2_1 ON t1.k1 = T2_1.ak2 INNER JOIN t2 T2_2 ON T2_1.k1 = T2_2.k1 " + "WHERE T2_2.ak2 = ?", (res) -> assertPartitions( - parititon("t2", "1") + partition("t2", "1") ), "1" ); @@ -639,7 +657,7 @@ public void testOuterJoin() { execute("SELECT * FROM t1 LEFT OUTER JOIN t2 T2_1 ON t1.k1 = T2_1.ak2 INNER JOIN t2 T2_2 ON t1.k1 = T2_2.ak2 " + "WHERE T2_1.ak2 = ? AND T2_2.ak2=?", (res) -> assertPartitions( - parititon("t2", "2") + partition("t2", "2") ), "1", "2" ); @@ -656,14 +674,14 @@ public void testSelfJoin() { execute("SELECT * FROM t1 A INNER JOIN t1 B ON A.k1 = B.k1 WHERE A.k1 = ?", (res) -> assertPartitions( - parititon("t1", "1") + partition("t1", "1") ), "1" ); execute("SELECT * FROM t1 A INNER JOIN t1 B ON A.k1 = B.k1 WHERE A.k1 = ? AND B.k1 = ?", (res) -> assertPartitions( - parititon("t1", "1") + partition("t1", "1") ), "1", "1" ); @@ -675,8 +693,8 @@ public void testSelfJoin() { execute("SELECT * FROM t1 A INNER JOIN t1 B ON A.k1 = B.k1 WHERE A.k1 = ? OR B.k1 = ?", (res) -> assertPartitions( - parititon("t1", "1"), - parititon("t1", "2") + partition("t1", "1"), + partition("t1", "2") ), "1", "2" ); @@ -863,6 +881,16 @@ private List> executeSingle(String sql, Object... args) { if (args != null && args.length > 0) qry.setArgs(args); + return executeSqlFieldsQuery(qry); + } + + /** + * Execute prepared SQL fields query. + * + * @param qry Query. + * @return Result. + */ + private List> executeSqlFieldsQuery(SqlFieldsQuery qry) { return client().context().query().querySqlFields(qry, false).getAll(); } @@ -928,7 +956,7 @@ private static void assertNoRequests() { * @param key Key. * @return Partition. */ - private int parititon(String cacheName, Object key) { + private int partition(String cacheName, Object key) { return client().affinity(cacheName).partition(key); } From c828979f8fc7a5b55eb628c33449f3c7e5424633 Mon Sep 17 00:00:00 2001 From: devozerov Date: Wed, 16 Jan 2019 17:06:51 +0300 Subject: [PATCH 69/80] More tests. --- .../query/h2/twostep/JoinPartitionPruningSelfTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index 43465cbe93b36..80ba6286d2136 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -716,6 +716,7 @@ private void createPartitionedTable(String name, Object... cols) { * @param name Name. * @param cols Columns. */ + @SuppressWarnings("SameParameterValue") private void createReplicatedTable(String name, Object... cols) { createTable0(name, true, cols); } From 4afd4c430f55c2335eee99366a5720694c0f4cac Mon Sep 17 00:00:00 2001 From: devozerov Date: Wed, 16 Jan 2019 18:29:28 +0300 Subject: [PATCH 70/80] Test fixes. --- .../processors/query/h2/affinity/PartitionExtractor.java | 3 ++- .../processors/query/h2/twostep/GridReduceQueryExecutor.java | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java index 822107fd230ed..bf57b1c1580a7 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java @@ -203,7 +203,7 @@ private List prepareTableModel0(GridSqlAst from, PartitionTableM PartitionTable tbl = prepareTable(from, model); - return Collections.singletonList(tbl); + return tbl != null ? Collections.singletonList(tbl) : Collections.emptyList(); } /** @@ -291,6 +291,7 @@ private static PartitionTable prepareTable(GridSqlAst from, PartitionTableModel String affColName = null; String secondAffColName = null; + // TODO: Sync! for (Column col : tbl0.getColumns()) { if (tbl0.isColumnForPartitionPruningStrict(col)) { if (affColName == null) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java index 62953ec54e51c..15788f2b7ddba 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java @@ -393,9 +393,12 @@ public Iterator> query( int timeoutMillis, GridQueryCancel cancel, Object[] params, - final int[] parts, + int[] parts, boolean lazy, MvccQueryTracker mvccTracker) { + if (qry.isLocal() && parts != null) + parts = null; + assert !qry.mvccEnabled() || mvccTracker != null; if (F.isEmpty(params)) From 2576de3dc8ce3c6212b63dcc9e87716fff447a22 Mon Sep 17 00:00:00 2001 From: devozerov Date: Thu, 17 Jan 2019 11:09:29 +0300 Subject: [PATCH 71/80] Thread-safe columns. --- .../processors/query/h2/opt/GridH2Table.java | 35 ++++++++++++++----- .../query/h2/sql/GridSqlQuerySplitter.java | 13 +++++-- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java index 91503772618fa..40c5fdd415108 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java @@ -121,6 +121,9 @@ public class GridH2Table extends TableBase { /** Flag remove index or not when table will be destroyed. */ private volatile boolean rmIndex; + /** Columns with thread-safe access. */ + private volatile Column[] safeColumns; + /** * Creates table. * @@ -1038,12 +1041,14 @@ public void addColumns(List cols, boolean ifNotExists) { lock(true); try { - int pos = columns.length; + Column[] safeColumns0 = safeColumns; + + int pos = safeColumns0.length; - Column[] newCols = new Column[columns.length + cols.size()]; + Column[] newCols = new Column[safeColumns0.length + cols.size()]; // First, let's copy existing columns to new array - System.arraycopy(columns, 0, newCols, 0, columns.length); + System.arraycopy(safeColumns0, 0, newCols, 0, safeColumns0.length); // And now, let's add new columns for (QueryField col : cols) { @@ -1084,13 +1089,16 @@ public void addColumns(List cols, boolean ifNotExists) { * @param cols Columns. * @param ifExists If EXISTS flag. */ + @SuppressWarnings("ForLoopReplaceableByForEach") public void dropColumns(List cols, boolean ifExists) { assert !ifExists || cols.size() == 1; lock(true); try { - int size = columns.length; + Column[] safeColumns0 = safeColumns; + + int size = safeColumns0.length; for (String name : cols) { if (!doesColumnExist(name)) { @@ -1110,8 +1118,8 @@ public void dropColumns(List cols, boolean ifExists) { int dst = 0; - for (int i = 0; i < columns.length; i++) { - Column column = columns[i]; + for (int i = 0; i < safeColumns0.length; i++) { + Column column = safeColumns0[i]; for (String name : cols) { if (F.eq(name, column.getName())) { @@ -1141,8 +1149,17 @@ public void dropColumns(List cols, boolean ifExists) { } } + @Override + protected void setColumns(Column[] columns) { + this.safeColumns = columns; + + super.setColumns(columns); + } + /** {@inheritDoc} */ @Override public Column[] getColumns() { + Column[] safeColumns0 = safeColumns; + Boolean insertHack = INSERT_HACK.get(); if (insertHack != null && insertHack) { @@ -1151,15 +1168,15 @@ public void dropColumns(List cols, boolean ifExists) { StackTraceElement elem = elems[2]; if (F.eq(elem.getClassName(), Insert.class.getName()) && F.eq(elem.getMethodName(), "prepare")) { - Column[] columns0 = new Column[columns.length - 3]; + Column[] columns0 = new Column[safeColumns0.length - 3]; - System.arraycopy(columns, 3, columns0, 0, columns0.length); + System.arraycopy(safeColumns0, 3, columns0, 0, columns0.length); return columns0; } } - return columns; + return safeColumns0; } /** diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java index fd88cbde3483c..818777c7015e0 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java @@ -128,6 +128,9 @@ public class GridSqlQuerySplitter { /** */ private boolean collocatedGrpBy; + /** */ + private boolean distributedJoins; + /** */ private IdentityHashMap uniqueFromAliases = new IdentityHashMap<>(); @@ -137,11 +140,14 @@ public class GridSqlQuerySplitter { /** * @param params Query parameters. * @param collocatedGrpBy If it is a collocated GROUP BY query. + * @param distributedJoins Distributed joins flag. * @param extractor Partition extractor. */ - public GridSqlQuerySplitter(Object[] params, boolean collocatedGrpBy, PartitionExtractor extractor) { + public GridSqlQuerySplitter(Object[] params, boolean collocatedGrpBy, boolean distributedJoins, + PartitionExtractor extractor) { this.params = params; this.collocatedGrpBy = collocatedGrpBy; + this.distributedJoins = distributedJoins; this.extractor = extractor; } @@ -206,7 +212,8 @@ public static GridCacheTwoStepQuery split( qry.explain(false); - GridSqlQuerySplitter splitter = new GridSqlQuerySplitter(params, collocatedGrpBy, h2.partitionExtractor()); + GridSqlQuerySplitter splitter = new GridSqlQuerySplitter(params, collocatedGrpBy, distributedJoins, + h2.partitionExtractor()); // Normalization will generate unique aliases for all the table filters in FROM. // Also it will collect all tables and schemas from the query. @@ -1548,7 +1555,7 @@ private void splitSelect( map.partitioned(hasPartitionedTables(mapQry)); map.hasSubQueries(hasSubQueries); - if (map.isPartitioned()) + if (map.isPartitioned() && !distributedJoins) map.derivedPartitions(extractor.extract(mapQry)); mapSqlQrys.add(map); From f94dc969c8df537747630cd3889b3dacb9f4498c Mon Sep 17 00:00:00 2001 From: devozerov Date: Thu, 17 Jan 2019 11:10:09 +0300 Subject: [PATCH 72/80] Removed TODO. --- .../processors/query/h2/affinity/PartitionExtractor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java index bf57b1c1580a7..db06f3ac7e43f 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java @@ -291,7 +291,6 @@ private static PartitionTable prepareTable(GridSqlAst from, PartitionTableModel String affColName = null; String secondAffColName = null; - // TODO: Sync! for (Column col : tbl0.getColumns()) { if (tbl0.isColumnForPartitionPruningStrict(col)) { if (affColName == null) From ffbbf5346b3506645be9a4f16d501530ef2a1639 Mon Sep 17 00:00:00 2001 From: devozerov Date: Thu, 17 Jan 2019 11:31:46 +0300 Subject: [PATCH 73/80] Preparing for config tests. --- .../twostep/JoinPartitionPruningSelfTest.java | 172 ++++++++++++++++++ 1 file changed, 172 insertions(+) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index 80ba6286d2136..35b3bb465a4c1 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -20,14 +20,24 @@ import org.apache.ignite.Ignite; import org.apache.ignite.IgniteException; import org.apache.ignite.binary.BinaryObject; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.QueryEntity; +import org.apache.ignite.cache.affinity.AffinityKeyMapped; +import org.apache.ignite.cache.affinity.AffinityKeyMapper; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.cache.query.SqlFieldsQuery; +import org.apache.ignite.cache.query.annotations.QuerySqlField; import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.managers.communication.GridIoMessage; import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2QueryRequest; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.lang.IgniteInClosure; +import org.apache.ignite.lang.IgnitePredicate; import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; @@ -40,6 +50,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -66,13 +77,23 @@ public class JoinPartitionPruningSelfTest extends GridCommonAbstractTest { /** Client node name. */ private static final String CLI_NAME = "cli"; + /** Memory. */ + private static final String REGION_MEM = "mem"; + + /** Disk. */ + private static final String REGION_DISK = "disk"; + /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { + cleanPersistenceDir(); + startGrid(getConfiguration("srv1")); startGrid(getConfiguration("srv2")); startGrid(getConfiguration("srv3")); startGrid(getConfiguration(CLI_NAME).setClientMode(true)); + + client().cluster().active(true); } /** {@inheritDoc} */ @@ -87,6 +108,8 @@ public class JoinPartitionPruningSelfTest extends GridCommonAbstractTest { /** {@inheritDoc} */ @Override protected void afterTestsStopped() throws Exception { stopAllGrids(); + + cleanPersistenceDir(); } /** {@inheritDoc} */ @@ -98,6 +121,15 @@ public class JoinPartitionPruningSelfTest extends GridCommonAbstractTest { res.setLocalHost("127.0.0.1"); + DataRegionConfiguration memRegion = + new DataRegionConfiguration().setName(REGION_MEM).setPersistenceEnabled(false); + + DataRegionConfiguration diskRegion = + new DataRegionConfiguration().setName(REGION_DISK).setPersistenceEnabled(true); + + res.setDataStorageConfiguration(new DataStorageConfiguration().setDataRegionConfigurations(diskRegion) + .setDefaultDataRegionConfiguration(memRegion)); + return res; } @@ -571,6 +603,80 @@ public void testJoinWithDifferentAffinityFunctions() { // With and without persistence. // TODO + + // Different affinity key mappers? + // TODO + } + + @SuppressWarnings("unchecked") + private void checkAffinityFunctions(CacheConfiguration ccfg1, CacheConfiguration ccfg2, boolean compatible) { + // Destroy old caches. + Ignite cli = client(); + + cli.destroyCaches(cli.cacheNames()); + + // Start new caches. + QueryEntity entity1 = new QueryEntity(KeyClass1.class, ValueClass.class).setTableName("t1"); + QueryEntity entity2 = new QueryEntity(KeyClass2.class, ValueClass.class).setTableName("t2"); + + ccfg1.setQueryEntities(Collections.singletonList(entity1)); + ccfg2.setQueryEntities(Collections.singletonList(entity2)); + + client().createCache(ccfg1); + client().createCache(ccfg2); + + // Conduct tests. + // TODO. + } + + /** + * Create custom cache configuration. + * + * @param name Name. + * @param parts Partitions. + * @param backups Backups. + * @param customAffinity Custom affinity function flag. + * @param customAffinityMapper Custom affinity key mapper flag. + * @param nodeFilter Whether to set node filter. + * @param persistent Whether to enable persistence. + * @return Cache configuration. + */ + private static CacheConfiguration cacheConfiguration( + String name, + int parts, + int backups, + boolean customAffinity, + boolean customAffinityMapper, + boolean nodeFilter, + boolean persistent + ) { + CacheConfiguration ccfg = new CacheConfiguration(); + + ccfg.setName(name); + ccfg.setCacheMode(CacheMode.PARTITIONED); + ccfg.setBackups(backups); + + RendezvousAffinityFunction affFunc; + + if (customAffinity) + affFunc = new CustomRendezvousAffinityFunction(); + else + affFunc = new RendezvousAffinityFunction(); + + affFunc.setPartitions(parts); + + ccfg.setAffinity(affFunc); + + if (customAffinityMapper) + ccfg.setAffinityMapper(new CustomAffinityMapper()); + + if (nodeFilter) + ccfg.setNodeFilter(new CustomNodeFilter()); + + if (persistent) + ccfg.setDataRegionName(REGION_DISK); + + return ccfg; } /** @@ -1050,4 +1156,70 @@ public boolean affinity() { return aff; } } + + /** + * Custom affinity function. + */ + private static class CustomRendezvousAffinityFunction extends RendezvousAffinityFunction { + // No-op. + } + + /** + * Custom affinity mapper. + */ + private static class CustomAffinityMapper implements AffinityKeyMapper { + /** {@inheritDoc} */ + @Override public Object affinityKey(Object key) { + return key; + } + + /** {@inheritDoc} */ + @Override public void reset() { + // No-op. + } + } + + /** + * Custom node filter. + */ + private static class CustomNodeFilter implements IgnitePredicate { + @Override public boolean apply(ClusterNode clusterNode) { + return true; + } + } + + /** + * Key class 1. + */ + @SuppressWarnings("unused") + private static class KeyClass1 { + /** Key. */ + @QuerySqlField + private long k1; + } + + /** + * Key class 2. + */ + @SuppressWarnings("unused") + private static class KeyClass2 { + /** Key. */ + @QuerySqlField + private long k1; + + /** Affinity key. */ + @QuerySqlField + @AffinityKeyMapped + private long ak2; + } + + /** + * Value class. + */ + @SuppressWarnings("unused") + private static class ValueClass { + /** Value. */ + @QuerySqlField + private long v; + } } From 1d1c8ffaf152df8322283a97be173c7fb576d1bc Mon Sep 17 00:00:00 2001 From: devozerov Date: Thu, 17 Jan 2019 11:50:31 +0300 Subject: [PATCH 74/80] WIP on config tests. --- .../twostep/JoinPartitionPruningSelfTest.java | 81 ++++++++++++------- 1 file changed, 50 insertions(+), 31 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index 35b3bb465a4c1..cf5b6ccbe0161 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -23,7 +23,6 @@ import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.QueryEntity; import org.apache.ignite.cache.affinity.AffinityKeyMapped; -import org.apache.ignite.cache.affinity.AffinityKeyMapper; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.cache.query.annotations.QuerySqlField; @@ -34,6 +33,7 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.managers.communication.GridIoMessage; +import org.apache.ignite.internal.processors.query.QueryUtils; import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2QueryRequest; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.lang.IgniteInClosure; @@ -590,6 +590,20 @@ public void testJoinWithReplicated() { @Test public void testJoinWithDifferentAffinityFunctions() { // Partition count. + // TODO: Doesn't work + checkAffinityFunctions( + cacheConfiguration(256, 1, false, false, false), + cacheConfiguration(256, 1, false, false, false), + true + ); + + checkAffinityFunctions( + cacheConfiguration(1024, 1, false, false, false), + cacheConfiguration(256, 1, false, false, false), + false + ); + + // Backups. // TODO // Different affinity functions. @@ -598,14 +612,8 @@ public void testJoinWithDifferentAffinityFunctions() { // Node filters. // TODO - // Backups. - // TODO - // With and without persistence. // TODO - - // Different affinity key mappers? - // TODO } @SuppressWarnings("unchecked") @@ -616,43 +624,72 @@ private void checkAffinityFunctions(CacheConfiguration ccfg1, CacheConfiguration cli.destroyCaches(cli.cacheNames()); // Start new caches. + ccfg1.setName("t1"); + ccfg2.setName("t2"); + QueryEntity entity1 = new QueryEntity(KeyClass1.class, ValueClass.class).setTableName("t1"); QueryEntity entity2 = new QueryEntity(KeyClass2.class, ValueClass.class).setTableName("t2"); ccfg1.setQueryEntities(Collections.singletonList(entity1)); ccfg2.setQueryEntities(Collections.singletonList(entity2)); + ccfg1.setSqlSchema(QueryUtils.DFLT_SCHEMA); + ccfg2.setSqlSchema(QueryUtils.DFLT_SCHEMA); + client().createCache(ccfg1); client().createCache(ccfg2); // Conduct tests. - // TODO. + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ?", + (res) -> assertPartitions( + partition("t1", "1") + ), + "1" + ); + + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t2.ak2 = ?", + (res) -> assertPartitions( + partition("t2", "2") + ), + "2" + ); + + if (compatible) { + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? OR t2.ak2 = ?", + (res) -> assertPartitions( + partition("t1", "1"), + partition("t2", "2") + ), + "1", "2" + ); + } + else { + execute("SELECT * FROM t1 INNER JOIN t2 ON t1.k1 = t2.ak2 WHERE t1.k1 = ? OR t2.ak2 = ?", + (res) -> assertNoPartitions(), + "1", "2" + ); + } } /** * Create custom cache configuration. * - * @param name Name. * @param parts Partitions. * @param backups Backups. * @param customAffinity Custom affinity function flag. - * @param customAffinityMapper Custom affinity key mapper flag. * @param nodeFilter Whether to set node filter. * @param persistent Whether to enable persistence. * @return Cache configuration. */ private static CacheConfiguration cacheConfiguration( - String name, int parts, int backups, boolean customAffinity, - boolean customAffinityMapper, boolean nodeFilter, boolean persistent ) { CacheConfiguration ccfg = new CacheConfiguration(); - ccfg.setName(name); ccfg.setCacheMode(CacheMode.PARTITIONED); ccfg.setBackups(backups); @@ -667,9 +704,6 @@ private static CacheConfiguration cacheConfiguration( ccfg.setAffinity(affFunc); - if (customAffinityMapper) - ccfg.setAffinityMapper(new CustomAffinityMapper()); - if (nodeFilter) ccfg.setNodeFilter(new CustomNodeFilter()); @@ -1164,21 +1198,6 @@ private static class CustomRendezvousAffinityFunction extends RendezvousAffinity // No-op. } - /** - * Custom affinity mapper. - */ - private static class CustomAffinityMapper implements AffinityKeyMapper { - /** {@inheritDoc} */ - @Override public Object affinityKey(Object key) { - return key; - } - - /** {@inheritDoc} */ - @Override public void reset() { - // No-op. - } - } - /** * Custom node filter. */ From 1987ff05db22ef1ccec4739ea5a696479707dfc0 Mon Sep 17 00:00:00 2001 From: devozerov Date: Thu, 17 Jan 2019 11:57:28 +0300 Subject: [PATCH 75/80] First config tests work now. --- .../h2/twostep/JoinPartitionPruningSelfTest.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index cf5b6ccbe0161..e5017eabc9c98 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -20,9 +20,9 @@ import org.apache.ignite.Ignite; import org.apache.ignite.IgniteException; import org.apache.ignite.binary.BinaryObject; +import org.apache.ignite.cache.CacheKeyConfiguration; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.QueryEntity; -import org.apache.ignite.cache.affinity.AffinityKeyMapped; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.cache.query.annotations.QuerySqlField; @@ -590,7 +590,6 @@ public void testJoinWithReplicated() { @Test public void testJoinWithDifferentAffinityFunctions() { // Partition count. - // TODO: Doesn't work checkAffinityFunctions( cacheConfiguration(256, 1, false, false, false), cacheConfiguration(256, 1, false, false, false), @@ -633,6 +632,9 @@ private void checkAffinityFunctions(CacheConfiguration ccfg1, CacheConfiguration ccfg1.setQueryEntities(Collections.singletonList(entity1)); ccfg2.setQueryEntities(Collections.singletonList(entity2)); + ccfg1.setKeyConfiguration(new CacheKeyConfiguration(entity1.getKeyType(), "k1")); + ccfg2.setKeyConfiguration(new CacheKeyConfiguration(entity2.getKeyType(), "ak2")); + ccfg1.setSqlSchema(QueryUtils.DFLT_SCHEMA); ccfg2.setSqlSchema(QueryUtils.DFLT_SCHEMA); @@ -1214,7 +1216,7 @@ private static class CustomNodeFilter implements IgnitePredicate { private static class KeyClass1 { /** Key. */ @QuerySqlField - private long k1; + private String k1; } /** @@ -1224,12 +1226,11 @@ private static class KeyClass1 { private static class KeyClass2 { /** Key. */ @QuerySqlField - private long k1; + private String k1; /** Affinity key. */ @QuerySqlField - @AffinityKeyMapped - private long ak2; + private String ak2; } /** @@ -1239,6 +1240,6 @@ private static class KeyClass2 { private static class ValueClass { /** Value. */ @QuerySqlField - private long v; + private String v; } } From 742b3a40276b3f4fc14612ec528aa9c77134be48 Mon Sep 17 00:00:00 2001 From: devozerov Date: Thu, 17 Jan 2019 12:00:09 +0300 Subject: [PATCH 76/80] WIP. --- .../twostep/JoinPartitionPruningSelfTest.java | 42 +++++++++++++++++-- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index e5017eabc9c98..5f86ac83ad14c 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -603,13 +603,49 @@ public void testJoinWithDifferentAffinityFunctions() { ); // Backups. - // TODO + checkAffinityFunctions( + cacheConfiguration(256, 1, false, false, false), + cacheConfiguration(256, 2, false, false, false), + true + ); // Different affinity functions. - // TODO + checkAffinityFunctions( + cacheConfiguration(256, 2, true, false, false), + cacheConfiguration(256, 2, false, false, false), + false + ); + + checkAffinityFunctions( + cacheConfiguration(256, 2, false, false, false), + cacheConfiguration(256, 2, true, false, false), + false + ); + + checkAffinityFunctions( + cacheConfiguration(256, 2, true, false, false), + cacheConfiguration(256, 2, true, false, false), + false + ); // Node filters. - // TODO + checkAffinityFunctions( + cacheConfiguration(256, 2, false, true, false), + cacheConfiguration(256, 2, false, false, false), + false + ); + + checkAffinityFunctions( + cacheConfiguration(256, 2, false, false, false), + cacheConfiguration(256, 2, false, true, false), + false + ); + + checkAffinityFunctions( + cacheConfiguration(256, 2, false, true, false), + cacheConfiguration(256, 2, false, true, false), + false + ); // With and without persistence. // TODO From 0bc01825140b53d1cedafb68c446b03ab63f1d5b Mon Sep 17 00:00:00 2001 From: devozerov Date: Thu, 17 Jan 2019 12:04:40 +0300 Subject: [PATCH 77/80] Done with tests. --- .../query/h2/affinity/PartitionExtractor.java | 3 ++- .../PartitionTableAffinityDescriptor.java | 13 ++++++++-- .../twostep/JoinPartitionPruningSelfTest.java | 24 ++++++++++++++++++- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java index db06f3ac7e43f..a9cc245488c47 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java @@ -349,7 +349,8 @@ private static PartitionTableAffinityDescriptor affinityForCache(CacheConfigurat return new PartitionTableAffinityDescriptor( aff, ccfg.getAffinity().partitions(), - hasNodeFilter + hasNodeFilter, + ccfg.getDataRegionName() ); } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTableAffinityDescriptor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTableAffinityDescriptor.java index 8389f06b182e7..21dab9cbea468 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTableAffinityDescriptor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTableAffinityDescriptor.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.query.h2.affinity; +import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.S; import java.io.Serializable; @@ -37,21 +38,27 @@ public class PartitionTableAffinityDescriptor implements Serializable { /** Whether node filter is set. */ private final boolean hasNodeFilter; + /** Data region name. */ + private final String dataRegion; + /** * Constructor. * * @param affFunc Affinity function type. * @param parts Number of partitions. * @param hasNodeFilter Whether node filter is set. + * @param dataRegion Data region. */ public PartitionTableAffinityDescriptor( PartitionAffinityFunctionType affFunc, int parts, - boolean hasNodeFilter + boolean hasNodeFilter, + String dataRegion ) { this.affFunc = affFunc; this.parts = parts; this.hasNodeFilter = hasNodeFilter; + this.dataRegion = dataRegion; } /** @@ -60,6 +67,7 @@ public PartitionTableAffinityDescriptor( * @param other Other descriptor. * @return {@code True} if compatible. */ + @SuppressWarnings("BooleanMethodIsAlwaysInverted") public boolean isCompatible(PartitionTableAffinityDescriptor other) { if (other == null) return false; @@ -74,7 +82,8 @@ public boolean isCompatible(PartitionTableAffinityDescriptor other) { return other.affFunc == PartitionAffinityFunctionType.RENDEZVOUS && !other.hasNodeFilter && - other.parts == parts; + other.parts == parts && + F.eq(other.dataRegion, dataRegion); } } diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java index 5f86ac83ad14c..1429f3f062a11 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinPartitionPruningSelfTest.java @@ -602,6 +602,12 @@ public void testJoinWithDifferentAffinityFunctions() { false ); + checkAffinityFunctions( + cacheConfiguration(256, 1, false, false, false), + cacheConfiguration(1024, 1, false, false, false), + false + ); + // Backups. checkAffinityFunctions( cacheConfiguration(256, 1, false, false, false), @@ -648,7 +654,23 @@ public void testJoinWithDifferentAffinityFunctions() { ); // With and without persistence. - // TODO + checkAffinityFunctions( + cacheConfiguration(256, 2, false, false, true), + cacheConfiguration(256, 2, false, false, false), + false + ); + + checkAffinityFunctions( + cacheConfiguration(256, 2, false, false, false), + cacheConfiguration(256, 2, false, false, true), + false + ); + + checkAffinityFunctions( + cacheConfiguration(256, 2, false, false, true), + cacheConfiguration(256, 2, false, false, true), + true + ); } @SuppressWarnings("unchecked") From bcb2e79699533e440f85b3e295930151db629763 Mon Sep 17 00:00:00 2001 From: devozerov Date: Thu, 17 Jan 2019 18:24:16 +0300 Subject: [PATCH 78/80] Merge with master. --- .../query/h2/affinity/PartitionExtractor.java | 24 ++++++++++++------- .../query/h2/sql/GridSqlColumn.java | 7 ------ ...weenOperationExtractPartitionSelfTest.java | 18 -------------- 3 files changed, 15 insertions(+), 34 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java index acbe256345143..0898e63b85348 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionExtractor.java @@ -41,15 +41,13 @@ import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQuery; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlSelect; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlTable; +import org.apache.ignite.internal.util.typedef.F; import org.h2.table.Column; import org.h2.value.Value; import org.jetbrains.annotations.Nullable; import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; /** * Partition tree extractor. @@ -429,8 +427,7 @@ private PartitionNode extractFromAnd(GridSqlOperation op, PartitionTableModel tb throws IgniteCheckedException { assert op.size() == 2; - // TODO: Pass model? - PartitionNode betweenNodes = tryExtractBetween(op); + PartitionNode betweenNodes = tryExtractBetween(op, tblModel); if (betweenNodes != null) return betweenNodes; @@ -644,12 +641,14 @@ else if (rightParam != null) * Try to extract partitions from {@code op} assuming that it's between operation or simple range. * * @param op Sql operation. + * @param tblModel Table model. * @return {@code PartitionSingleNode} if operation reduced to one partition, * {@code PartitionGroupNode} if operation reduced to multiple partitions or null if operation is neither * between nor simple range. Null also returns if it's not possible to extract partitions from given operation. * @throws IgniteCheckedException If failed. */ - private PartitionNode tryExtractBetween(GridSqlOperation op) throws IgniteCheckedException { + private PartitionNode tryExtractBetween(GridSqlOperation op, PartitionTableModel tblModel) + throws IgniteCheckedException { // Between operation (or similar range) should contain exact two children. assert op.size() == 2; @@ -740,11 +739,18 @@ else if (!((GridSqlOperationType.BIGGER == leftOpType || GridSqlOperationType.BI Set parts = new HashSet<>(); - PartitionTableDescriptor desc = descriptor(tbl); + PartitionTable tbl0 = tblModel.table(leftCol.tableAlias()); + + // If table is in ignored set, then we cannot use it for partition extraction. + if (tbl0 == null) + return null; for (long i = leftLongVal; i <= rightLongVal; i++) { - parts.add(new PartitionConstantNode(desc, - idx.kernalContext().affinity().partition((tbl).cacheName(), i))); + Object constVal = H2Utils.convert(i, idx, leftCol.column().getType()); + + int part = idx.kernalContext().affinity().partition(tbl0.cacheName(), constVal); + + parts.add(new PartitionConstantNode(tbl0, part)); if (parts.size() > maxPartsCntBetween) return null; diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlColumn.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlColumn.java index 7dd93dfcfe2f6..9c4d8c0057d15 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlColumn.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlColumn.java @@ -85,13 +85,6 @@ public String schema() { return schema; } - /** - * @return Table alias. - */ - public String tableAlias() { - return tblAlias; - } - /** * @param tblAlias Table alias. */ diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/BetweenOperationExtractPartitionSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/BetweenOperationExtractPartitionSelfTest.java index fbdbfb0ab6cf9..f6e73bcdca630 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/BetweenOperationExtractPartitionSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/BetweenOperationExtractPartitionSelfTest.java @@ -27,7 +27,6 @@ import java.util.stream.Collectors; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteException; -import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.QueryEntity; import org.apache.ignite.cache.query.FieldsQueryCursor; @@ -41,7 +40,6 @@ import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.spi.IgniteSpiException; import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; -import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.Test; import org.junit.runner.RunWith; @@ -480,22 +478,6 @@ public void testBetweenPartitionsDefaultLimitExceeding() { testBetweenConstOperator(BETWEEN_QRY, 1, 17, 17, EMPTY_PARTITIONS_ARRAY); } - /** - * Check custom partitions limit exceeding. - */ - @Test - public void testBetweenPartitionsCustomLimitExceeding() { - try (GridTestUtils.SystemProperty ignored = new GridTestUtils. - SystemProperty(IgniteSystemProperties.IGNITE_SQL_MAX_EXTRACTED_PARTS_FROM_BETWEEN, "4")){ - - // Default limit (16) not exceeded. - testBetweenConstOperator(BETWEEN_QRY, 1, 4, 4); - - // Default limit (16) exceeded. - testBetweenConstOperator(BETWEEN_QRY, 1, 5, 5, EMPTY_PARTITIONS_ARRAY); - } - } - /** * Check range expression with constant values. */ From 2726a30ac317f8cbcf06c33412cd9ccc93e8c45b Mon Sep 17 00:00:00 2001 From: devozerov Date: Thu, 17 Jan 2019 18:27:28 +0300 Subject: [PATCH 79/80] Minors. --- .../internal/processors/query/h2/IgniteH2Indexing.java | 3 ++- .../query/h2/affinity/PartitionAffinityFunctionType.java | 2 +- .../processors/query/h2/affinity/PartitionTable.java | 9 +-------- .../query/h2/affinity/PartitionTableModel.java | 4 ++-- .../internal/processors/query/h2/opt/GridH2Table.java | 2 +- 5 files changed, 7 insertions(+), 13 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index 998b8c2384968..5e00aeab77111 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -140,6 +140,7 @@ import org.apache.ignite.internal.util.GridBoundedConcurrentLinkedHashMap; import org.apache.ignite.internal.util.GridEmptyCloseableIterator; import org.apache.ignite.internal.util.GridSpinBusyLock; +import org.apache.ignite.internal.util.IgniteUtils; import org.apache.ignite.internal.util.lang.GridCloseableIterator; import org.apache.ignite.internal.util.lang.GridPlainRunnable; import org.apache.ignite.internal.util.lang.IgniteInClosure2X; @@ -2080,7 +2081,7 @@ else if (derivedParts != null) { Collection realParts = derivedParts.tree().apply(args); if (F.isEmpty(realParts)) - return new int[0]; + return IgniteUtils.EMPTY_INTS; else { int[] realParts0 = new int[realParts.size()]; diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionAffinityFunctionType.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionAffinityFunctionType.java index 4ddafff55c73f..85eb5a1f14022 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionAffinityFunctionType.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionAffinityFunctionType.java @@ -52,7 +52,7 @@ public int value() { * Get type from value. * * @param val Value. - * @return Type or [@code null} if cannot be resolved.. + * @return Type or {@code null} if cannot be resolved.. */ @Nullable public static PartitionAffinityFunctionType fromValue(int val) { for (PartitionAffinityFunctionType typ : values()) { diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTable.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTable.java index 8b14b7805a314..677d037457c63 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTable.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTable.java @@ -81,13 +81,6 @@ public String cacheName() { return cacheName; } - /** - * @return {@code True} if affinity oclumn exists. - */ - public boolean hasAffinityColumn() { - return affColName != null; - } - /** * Check whether passed column is affinity column. * @@ -108,7 +101,7 @@ public int joinGroup() { /** * @param joinGrp Join group index. */ - public void joinGorup(int joinGrp) { + public void joinGroup(int joinGrp) { this.joinGrp = joinGrp; } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTableModel.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTableModel.java index a2d118619ad07..63939411cf65a 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTableModel.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTableModel.java @@ -52,7 +52,7 @@ public class PartitionTableModel { public void addTable(PartitionTable tbl, PartitionTableAffinityDescriptor aff) { int grpIdx = grpIdxGen++; - tbl.joinGorup(grpIdx); + tbl.joinGroup(grpIdx); tbls.put(tbl.alias(), tbl); grps.put(grpIdx, new PartitionJoinGroup(aff).addTable(tbl)); @@ -130,7 +130,7 @@ public void addJoin(PartitionJoinCondition cond) { // Safe to merge groups. for (PartitionTable tbl : rightGrp.tables()) { - tbl.joinGorup(leftTbl.joinGroup()); + tbl.joinGroup(leftTbl.joinGroup()); leftGrp.addTable(tbl); } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java index 40c5fdd415108..d165672abf433 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java @@ -266,7 +266,7 @@ public boolean isColumnForPartitionPruning(Column col) { * SELECT * FROM dept d INNER JOIN emp e ON d.id = e._KEY WHERE e.dept_id=? AND d.id=? * * NB: The last query is not logically correct and will produce empty result. However, it is correct from SQL - * perspective, so we should make incorrect assumptions about partitions as ti may make situation even worse. + * perspective, so we should make incorrect assumptions about partitions as it may make situation even worse. * * @param col Column. * @return {@code True} if column could be used for partition extraction on both server and client sides and for From c31e5d8eb3c2d004413051856934d363664d8154 Mon Sep 17 00:00:00 2001 From: devozerov Date: Thu, 17 Jan 2019 18:33:17 +0300 Subject: [PATCH 80/80] Minors. --- .../affinity/PartitionAffinityFunctionType.java | 17 ----------------- .../query/h2/affinity/PartitionResult.java | 7 ------- .../query/h2/affinity/PartitionTable.java | 1 + .../processors/query/h2/opt/GridH2Table.java | 4 ++-- 4 files changed, 3 insertions(+), 26 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionAffinityFunctionType.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionAffinityFunctionType.java index 85eb5a1f14022..4c88fcbdd60da 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionAffinityFunctionType.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionAffinityFunctionType.java @@ -17,8 +17,6 @@ package org.apache.ignite.internal.processors.query.h2.affinity; -import org.jetbrains.annotations.Nullable; - /** * Affinity function type. */ @@ -47,19 +45,4 @@ public enum PartitionAffinityFunctionType { public int value() { return val; } - - /** - * Get type from value. - * - * @param val Value. - * @return Type or {@code null} if cannot be resolved.. - */ - @Nullable public static PartitionAffinityFunctionType fromValue(int val) { - for (PartitionAffinityFunctionType typ : values()) { - if (typ.val == val) - return typ; - } - - return null; - } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionResult.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionResult.java index 675b9b199eac7..daa14d3d63728 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionResult.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionResult.java @@ -56,13 +56,6 @@ public PartitionTableAffinityDescriptor affinity() { return aff; } - /** - * @return Join group. - */ - public int joinGroup() { - return tree.joinGroup(); - } - /** {@inheritDoc} */ @Override public String toString() { return S.toString(PartitionResult.class, this); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTable.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTable.java index 677d037457c63..1b996c15e2727 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTable.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/affinity/PartitionTable.java @@ -87,6 +87,7 @@ public String cacheName() { * @param colName Column name. * @return {@code True} if affinity column. */ + @SuppressWarnings("BooleanMethodIsAlwaysInverted") public boolean isAffinityColumn(String colName) { return F.eq(colName, affColName) || F.eq(colName, secondAffColName); } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java index d165672abf433..fff12e3d74f6b 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java @@ -1149,8 +1149,8 @@ public void dropColumns(List cols, boolean ifExists) { } } - @Override - protected void setColumns(Column[] columns) { + /** {@inheritDoc} */ + @Override protected void setColumns(Column[] columns) { this.safeColumns = columns; super.setColumns(columns);