From fe9df9bcc0b7b25bf05247e4999a75b65642b1fb Mon Sep 17 00:00:00 2001 From: sounakr Date: Tue, 24 Oct 2017 10:22:52 +0530 Subject: [PATCH] DataMap Access Path --- .../carbondata/core/datamap/DataMapMeta.java | 17 +- .../core/datamap/DataMapStoreManager.java | 33 ++- .../datamap/dev/DataMapAndNodeExpression.java | 58 +++++ .../core/datamap/dev/DataMapChooser.java | 84 +++++++ .../datamap/dev/DataMapColumnExpression.java | 55 +++++ .../dev/DataMapColumnNameNodeExpression.java | 45 ++++ .../dev/DataMapColumnNodeExpression.java | 51 ++++ .../datamap/dev/DataMapCoveringOptions.java | 40 ++++ .../dev/DataMapEqualNodeExpression.java | 41 ++++ .../core/datamap/dev/DataMapEvaluator.java | 143 ++++++++++++ .../core/datamap/dev/DataMapExpression.java | 41 ++++ .../dev/DataMapExpressionContainer.java | 32 +++ .../datamap/dev/DataMapExpressionTree.java | 219 ++++++++++++++++++ .../datamap/dev/DataMapExpressionType.java | 33 +++ .../core/datamap/dev/DataMapFilterNode.java | 38 +++ ...DataMapGreaterThanEqualNodeExpression.java | 41 ++++ .../dev/DataMapGreaterThanNodeExpression.java | 42 ++++ .../DataMapLessThanEqualNodeExpression.java | 42 ++++ .../dev/DataMapLessThanNodeExpression.java | 42 ++++ .../dev/DataMapLikeNodeExpression.java | 41 ++++ .../dev/DataMapLiteralNodeExpression.java | 52 +++++ .../datamap/dev/DataMapNodeExpression.java | 35 +++ .../hadoop/api/CarbonTableInputFormat.java | 6 +- .../api/DistributableDataMapFormat.java | 4 +- .../datamap/DataMapWriterSuite.scala | 7 +- .../datamap/DataMapWriterListener.java | 4 +- 26 files changed, 1225 insertions(+), 21 deletions(-) create mode 100644 core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapAndNodeExpression.java create mode 100644 core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapChooser.java create mode 100644 core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapColumnExpression.java create mode 100644 core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapColumnNameNodeExpression.java create mode 100644 core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapColumnNodeExpression.java create mode 100644 core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapCoveringOptions.java create mode 100644 core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapEqualNodeExpression.java create mode 100644 core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapEvaluator.java create mode 100644 core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapExpression.java create mode 100644 core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapExpressionContainer.java create mode 100644 core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapExpressionTree.java create mode 100644 core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapExpressionType.java create mode 100644 core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFilterNode.java create mode 100644 core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapGreaterThanEqualNodeExpression.java create mode 100644 core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapGreaterThanNodeExpression.java create mode 100644 core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapLessThanEqualNodeExpression.java create mode 100644 core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapLessThanNodeExpression.java create mode 100644 core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapLikeNodeExpression.java create mode 100644 core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapLiteralNodeExpression.java create mode 100644 core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapNodeExpression.java diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java index 7746acf6fe9..0de1658c5fb 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java @@ -18,25 +18,20 @@ package org.apache.carbondata.core.datamap; import java.util.List; +import java.util.Map; import org.apache.carbondata.core.indexstore.schema.FilterType; public class DataMapMeta { - private List indexedColumns; + private Map indexedColumnsAndOperations; - private FilterType optimizedOperation; - - public DataMapMeta(List indexedColumns, FilterType optimizedOperation) { - this.indexedColumns = indexedColumns; - this.optimizedOperation = optimizedOperation; + public DataMapMeta(Map indexedColumnsOptimized) { + this.indexedColumnsAndOperations = indexedColumnsOptimized; } - public List getIndexedColumns() { - return indexedColumns; + public Map getIndexedColumns() { + return indexedColumnsAndOperations; } - public FilterType getOptimizedOperation() { - return optimizedOperation; - } } diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java index 2b5d5cd370b..b7c82c18ce1 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java @@ -24,6 +24,8 @@ import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; +import org.apache.carbondata.core.datamap.dev.DataMapChooser; +import org.apache.carbondata.core.datamap.dev.DataMapExpression; import org.apache.carbondata.core.datamap.dev.DataMapFactory; import org.apache.carbondata.core.indexstore.BlockletDetailsFetcher; import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMap; @@ -31,6 +33,7 @@ import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.mutate.SegmentUpdateDetails; import org.apache.carbondata.core.mutate.UpdateVO; +import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager; /** @@ -65,7 +68,7 @@ public List getAllDataMap(AbsoluteTableIdentifier identifier) { * @param factoryClass * @return */ - public TableDataMap getDataMap(AbsoluteTableIdentifier identifier, + public TableDataMap getDataMap(AbsoluteTableIdentifier identifier, FilterResolverIntf resolver, String dataMapName, String factoryClass) { String table = identifier.uniqueName(); List tableDataMaps = allDataMaps.get(table); @@ -80,7 +83,8 @@ public TableDataMap getDataMap(AbsoluteTableIdentifier identifier, } } } else { - dataMap = getTableDataMap(dataMapName, tableDataMaps); + // dataMap = getTableDataMap(dataMapName, tableDataMaps); + dataMap = getTableDataMap(resolver, identifier, tableDataMaps, dataMapName); } if (dataMap == null) { @@ -123,6 +127,10 @@ public TableDataMap createAndRegisterDataMap(AbsoluteTableIdentifier identifier, LOGGER.error(e); throw new RuntimeException(e); } + + if (null != allDataMaps.get(table)) { + tableDataMaps.addAll(allDataMaps.get(table)); + } tableDataMaps.add(dataMap); allDataMaps.put(table, tableDataMaps); return dataMap; @@ -140,6 +148,24 @@ private TableDataMap getTableDataMap(String dataMapName, return dataMap; } + private TableDataMap getTableDataMap(FilterResolverIntf resolver, + AbsoluteTableIdentifier identifier, List tableDataMaps, String dataMapName) { + + DataMapExpression dataMapExpression = null; + DataMapChooser mapChooser = new DataMapChooser(resolver.getFilterExpression(), identifier); + dataMapExpression = mapChooser.dataMapSelection(); + + TableDataMap dataMap = null; + for (TableDataMap tableDataMap: tableDataMaps) { + if (tableDataMap.getDataMapName().equals(dataMapName)) { + dataMap = tableDataMap; + break; + } + } + return dataMap; + + } + /** * Clear the datamap/datamaps of a table from memory * @param identifier Table identifier @@ -162,12 +188,13 @@ public void clearDataMap(AbsoluteTableIdentifier identifier) { /** * Get the blocklet datamap factory to get the detail information of blocklets + * * @param identifier * @return */ private BlockletDetailsFetcher getBlockletDetailsFetcher(AbsoluteTableIdentifier identifier) { TableDataMap blockletMap = - getDataMap(identifier, BlockletDataMap.NAME, BlockletDataMapFactory.class.getName()); + getDataMap(identifier, null, BlockletDataMap.NAME, BlockletDataMapFactory.class.getName()); return (BlockletDetailsFetcher) blockletMap.getDataMapFactory(); } diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapAndNodeExpression.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapAndNodeExpression.java new file mode 100644 index 00000000000..a734a593789 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapAndNodeExpression.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.core.datamap.dev; + +import java.util.List; + +import org.apache.carbondata.core.indexstore.Blocklet; +import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; + +public class DataMapAndNodeExpression extends DataMapExpression { + protected DataMapExpression left; + protected DataMapExpression right; + + public DataMapAndNodeExpression (DataMapExpression left, DataMapExpression right) { + this.left = left; + this.right = right; + } + + public DataMapExpression getLeft () { + return left; + } + + public DataMapExpression getRight () { + return right; + } + + public void setLeft (DataMapExpression left) { + this.left = left; + } + + public void setRight (DataMapExpression right) { + this.right = right; + } + + @Override public DataMapExpressionType getDataMapExpressionType() { + return DataMapExpressionType.AND_DATAMAP; + } + + @Override public List prune(FilterResolverIntf filterExp) { + return null; + } + +} diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapChooser.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapChooser.java new file mode 100644 index 00000000000..a5ccd364884 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapChooser.java @@ -0,0 +1,84 @@ +package org.apache.carbondata.core.datamap.dev; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.carbondata.core.datamap.DataMapMeta; +import org.apache.carbondata.core.datamap.DataMapStoreManager; +import org.apache.carbondata.core.datamap.TableDataMap; +import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; +import org.apache.carbondata.core.scan.expression.Expression; + +public class DataMapChooser { + + private Expression srcExpression; + + private AbsoluteTableIdentifier identifier; + + public DataMapChooser(Expression srcExpression, AbsoluteTableIdentifier identifier) { + this.srcExpression = srcExpression; + this.identifier = identifier; + } + + public DataMapExpression dataMapSelection() { + DataMapCoveringOptions.DataMapCovered DataMapCovered; + Map> dataMapCoverOption = new HashMap<>(); + DataMapExpression dataMapExpression = null; + + DataMapCovered = dataMapConvered(dataMapCoverOption); + + if (DataMapCovered.equals(DataMapCoveringOptions.DataMapCovered.FULLY_COVERED) || DataMapCovered + .equals(DataMapCoveringOptions.DataMapCovered.PARTIALLY_COVERED)) { + // Only one data Map factory. Form the tree based on that. + DataMapExpressionTree dataMapExpressionTree = + new DataMapExpressionTree(DataMapCovered, dataMapCoverOption); + dataMapExpression = dataMapExpressionTree.dataMapExpressionTreeFormation(); + } + + return dataMapExpression; + } + + public DataMapCoveringOptions.DataMapCovered dataMapConvered( + Map> dataMapChoosen) { + // Get list of dataMaps. + List tableDataMaps = + DataMapStoreManager.getInstance().getAllDataMap(this.identifier); + + if (tableDataMaps != null) { + for (TableDataMap tableDataMap : tableDataMaps) { + DataMapFactory factory = tableDataMap.getDataMapFactory(); + + // DataMapColumnExpression dataMapColumnExpression = new DataMapColumnExpression(this.srcExpression); + Map> coverage = + checkCoverage(factory.getMeta()); + if (coverage.keySet().contains(DataMapCoveringOptions.DataMapCovered.FULLY_COVERED)) { + // Fully covered. So choose this dataMap. + dataMapChoosen.put(factory, new ArrayList<>( + coverage.get(DataMapCoveringOptions.DataMapCovered.FULLY_COVERED))); + return DataMapCoveringOptions.DataMapCovered.FULLY_COVERED; + } else if (coverage.keySet() + .contains(DataMapCoveringOptions.DataMapCovered.PARTIALLY_COVERED)) { + dataMapChoosen.put(factory, new ArrayList<>( + coverage.get(DataMapCoveringOptions.DataMapCovered.PARTIALLY_COVERED))); + } + } + } + if (dataMapChoosen.size() > 0) { + return DataMapCoveringOptions.DataMapCovered.PARTIALLY_COVERED; + } + return DataMapCoveringOptions.DataMapCovered.NOT_COVERED; + } + + + private Map> checkCoverage( + DataMapMeta meta) { + // Check if the coverage. + DataMapEvaluator dataMapEvaluator = new DataMapEvaluator(srcExpression); + return dataMapEvaluator.expressionEvaluator(meta); + } + + private void getDataMapList(AbsoluteTableIdentifier identifier) { + } +} diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapColumnExpression.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapColumnExpression.java new file mode 100644 index 00000000000..ad9eb4374fb --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapColumnExpression.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.core.datamap.dev; + +import org.apache.carbondata.core.indexstore.schema.FilterType; +import org.apache.carbondata.core.scan.expression.Expression; + +public class DataMapColumnExpression { + private Expression expr; + + private String ColumnName; + + private FilterType filterType; + + private boolean covered; + + public DataMapColumnExpression(Expression expr, String columnName, FilterType filterType, + boolean covered) { + this.expr = expr; + this.ColumnName = columnName; + this.filterType = filterType; + this.covered = covered; + } + + public Expression getExpr() { + return expr; + } + + public String getColumnName() { + return ColumnName; + } + + public FilterType getFilterType() { + return filterType; + } + + public boolean isCovered() { + return covered; + } +} diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapColumnNameNodeExpression.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapColumnNameNodeExpression.java new file mode 100644 index 00000000000..48acd95e4d3 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapColumnNameNodeExpression.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.core.datamap.dev; + +import java.util.List; + +import org.apache.carbondata.core.indexstore.Blocklet; +import org.apache.carbondata.core.metadata.datatype.DataType; +import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; + +public class DataMapColumnNameNodeExpression extends DataMapExpression { + protected String columnName; + + public DataMapColumnNameNodeExpression (String columnName) { + this.columnName = columnName; + } + + public String getColumnName () { + return columnName; + } + + @Override public DataMapExpressionType getDataMapExpressionType() { + return DataMapExpressionType.LITERAL_DATAMAP; + } + + @Override public List prune(FilterResolverIntf filterExp) { + return null; + } + +} diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapColumnNodeExpression.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapColumnNodeExpression.java new file mode 100644 index 00000000000..7fd41a1f6be --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapColumnNodeExpression.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.core.datamap.dev; + +import java.util.List; + +import org.apache.carbondata.core.indexstore.Blocklet; +import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; + +public class DataMapColumnNodeExpression extends DataMapExpression{ + protected List filterNodes; + + public DataMapColumnNodeExpression (List filterNodes){ + this.filterNodes = filterNodes; + } + + public List getFilterNodes () { + return filterNodes; + } + + public void addFilterNodes (DataMapFilterNode filterNode) { + filterNodes.add(filterNode); + } + + public void removeFilterNodes (DataMapFilterNode filterNode) { + filterNodes.remove(filterNode); + } + + @Override public DataMapExpressionType getDataMapExpressionType() { + return DataMapExpressionType.COLUMN_DATAMAP; + } + + @Override public List prune(FilterResolverIntf filterExp) { + return null; + } +} diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapCoveringOptions.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapCoveringOptions.java new file mode 100644 index 00000000000..7d7d34dacb1 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapCoveringOptions.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.core.datamap.dev; + +/* Data Map Covering Options */ + +public class DataMapCoveringOptions { + + public static DataMapCovered getDataMapCoveringOptions (String dataMapCoveringOptions) { + switch (dataMapCoveringOptions.toUpperCase()) { + case "NOT_COVERED": + return DataMapCovered.NOT_COVERED; + case "PARTIALLY_COVERED": + return DataMapCovered.PARTIALLY_COVERED; + case "FULLY_COVERED": + return DataMapCovered.FULLY_COVERED; + default: + return DataMapCovered.NOT_COVERED; + } + } + public enum DataMapCovered { + NOT_COVERED, PARTIALLY_COVERED, FULLY_COVERED; + } +} + diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapEqualNodeExpression.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapEqualNodeExpression.java new file mode 100644 index 00000000000..20346da7827 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapEqualNodeExpression.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.core.datamap.dev; + +import java.util.List; + +import org.apache.carbondata.core.datamap.DataMapMeta; +import org.apache.carbondata.core.indexstore.Blocklet; +import org.apache.carbondata.core.scan.expression.Expression; +import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; + +public class DataMapEqualNodeExpression extends DataMapFilterNode { + + public DataMapEqualNodeExpression (DataMapExpression columnExpr, DataMapExpression literal, + Expression expr, DataMapFactory dataMapFactory, DataMapMeta dataMapMeta) { + super(columnExpr, literal, expr, dataMapFactory, dataMapMeta); + } + + @Override public DataMapExpressionType getDataMapExpressionType() { + return DataMapExpressionType.EQUALS_DATAMAP_FILTERTYPE; + } + + @Override public List prune(FilterResolverIntf filterExp) { + return null; + } +} diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapEvaluator.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapEvaluator.java new file mode 100644 index 00000000000..183045b7c70 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapEvaluator.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.core.datamap.dev; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.carbondata.core.datamap.DataMapMeta; +import org.apache.carbondata.core.indexstore.schema.FilterType; +import org.apache.carbondata.core.scan.expression.ColumnExpression; +import org.apache.carbondata.core.scan.expression.Expression; +import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression; +import org.apache.carbondata.core.scan.expression.conditional.GreaterThanEqualToExpression; +import org.apache.carbondata.core.scan.expression.conditional.GreaterThanExpression; +import org.apache.carbondata.core.scan.expression.conditional.LessThanEqualToExpression; +import org.apache.carbondata.core.scan.expression.conditional.LessThanExpression; +import org.apache.carbondata.core.scan.filter.intf.ExpressionType; + +import org.apache.commons.collections.map.HashedMap; + +public class DataMapEvaluator { + + private Expression filterExpression; + + public DataMapEvaluator (Expression filterExpression) { + this.filterExpression = filterExpression; + } + + private ExpressionType matchFilterType (FilterType filterType) { + switch (filterType) { + case EQUALTO: + return ExpressionType.EQUALS; + case GREATER_THAN: + return ExpressionType.GREATERTHAN; + case GREATER_THAN_EQUAL: + return ExpressionType.GREATERTHAN_EQUALTO; + case LESS_THAN: + return ExpressionType.LESSTHAN; + case LESS_THAN_EQUAL: + return ExpressionType.LESSTHAN_EQUALTO; + case LIKE: + return ExpressionType.RANGE; + default: + return ExpressionType.UNKNOWN; + } + } + + private ExpressionType matchFilterOperator (Expression exp) { + if (exp instanceof EqualToExpression) { + return ExpressionType.EQUALS; + } else if (exp instanceof GreaterThanExpression) { + return ExpressionType.GREATERTHAN; + } else if (exp instanceof GreaterThanEqualToExpression) { + return ExpressionType.GREATERTHAN_EQUALTO; + } else if (exp instanceof LessThanExpression) { + return ExpressionType.LESSTHAN; + } else if (exp instanceof LessThanEqualToExpression) { + return ExpressionType.LESSTHAN_EQUALTO; + } else { + return ExpressionType.UNKNOWN; + } + } + + private boolean filterExpressionMap(Expression currentNode, Expression parentNode, + DataMapExpressionContainer validBlock, String columnName, FilterType filterType) { + if (null == currentNode) { + currentNode = filterExpression; + parentNode = filterExpression; + validBlock.setDataMapExpressionContainer(null); + } + + if (((currentNode instanceof ColumnExpression) && ((ColumnExpression) currentNode) + .getColumnName().equalsIgnoreCase(columnName)) && (matchFilterOperator(parentNode) + .equals(matchFilterType(filterType)))) { + validBlock.setDataMapExpressionContainer(parentNode); + return true; + } + + for (Expression exp : currentNode.getChildren()) { + if (null != exp) { + return filterExpressionMap(exp, currentNode, validBlock, columnName, filterType); + } else { + return false; + } + } + return false; + } + + public Map> expressionEvaluator( + DataMapMeta dataMapMeta) { + // Check each columns and its parent expression is there in the filterExpression or not. + Map dataMapColumns = dataMapMeta.getIndexedColumns(); + Map dataMapCoverMap = new HashedMap(); + Map dataMapCoverage = new HashedMap(); + Map> + dataMapCoverageFinalResult = new HashMap<>(); + Expression validBlock = null; + for (Map.Entry dataMapColumnFilter : dataMapColumns.entrySet()) { + DataMapExpressionContainer validExprBlock = new DataMapExpressionContainer(); + boolean covered = + filterExpressionMap(null, null, validExprBlock, dataMapColumnFilter.getKey(), + dataMapColumnFilter.getValue()); + DataMapColumnExpression dataMapColumnExpression = + new DataMapColumnExpression(validExprBlock.getExpressionBlock(), + dataMapColumnFilter.getKey(), dataMapColumnFilter.getValue(), covered); + dataMapCoverage.put(dataMapColumnFilter.getKey(), covered); + dataMapCoverMap.put(dataMapColumnFilter.getKey(), dataMapColumnExpression); + } + + // Check if all the values are set to true or partially true or all are false. + if (dataMapCoverage.values().containsAll(Arrays.asList(true))) { + dataMapCoverageFinalResult.put(DataMapCoveringOptions.DataMapCovered.FULLY_COVERED, + new ArrayList<>(dataMapCoverMap.values())); + return dataMapCoverageFinalResult; + } else if (dataMapCoverMap.values().containsAll(Arrays.asList(false))) { + dataMapCoverageFinalResult.put(DataMapCoveringOptions.DataMapCovered.NOT_COVERED, + new ArrayList<>(dataMapCoverMap.values())); + return dataMapCoverageFinalResult; + } else { + dataMapCoverageFinalResult.put(DataMapCoveringOptions.DataMapCovered.PARTIALLY_COVERED, + new ArrayList<>(dataMapCoverMap.values())); + return dataMapCoverageFinalResult; + } + } +} diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapExpression.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapExpression.java new file mode 100644 index 00000000000..ae7755ecb53 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapExpression.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.core.datamap.dev; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.indexstore.Blocklet; +import org.apache.carbondata.core.scan.expression.Expression; +import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; + +public abstract class DataMapExpression { + + protected List children = + new ArrayList(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); + + public abstract DataMapExpressionType getDataMapExpressionType(); + + public abstract List prune(FilterResolverIntf filterExp); + + public List getChildren() { + return children; + } + +} diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapExpressionContainer.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapExpressionContainer.java new file mode 100644 index 00000000000..cffce57dd70 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapExpressionContainer.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.core.datamap.dev; + +import org.apache.carbondata.core.scan.expression.Expression; + +public class DataMapExpressionContainer { + private Expression expressionBlock; + + public void setDataMapExpressionContainer (Expression expr) { + this.expressionBlock = expr; + } + + public Expression getExpressionBlock () { + return expressionBlock; + } +} diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapExpressionTree.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapExpressionTree.java new file mode 100644 index 00000000000..c62f79d3f07 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapExpressionTree.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.core.datamap.dev; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.carbondata.core.indexstore.Blocklet; +import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; + +import static org.apache.carbondata.core.indexstore.schema.FilterType.EQUALTO; +import static org.apache.carbondata.core.indexstore.schema.FilterType.GREATER_THAN; +import static org.apache.carbondata.core.indexstore.schema.FilterType.GREATER_THAN_EQUAL; +import static org.apache.carbondata.core.indexstore.schema.FilterType.LESS_THAN; +import static org.apache.carbondata.core.indexstore.schema.FilterType.LESS_THAN_EQUAL; +import static org.apache.carbondata.core.indexstore.schema.FilterType.LIKE; + +public class DataMapExpressionTree { + + private DataMapCoveringOptions.DataMapCovered coverage; + + private Map> dataMapFactoryListMap; + + public DataMapExpressionTree(DataMapCoveringOptions.DataMapCovered coverage, + Map> dataMapFactoryListMap) { + this.coverage = coverage; + this.dataMapFactoryListMap = dataMapFactoryListMap; + } + + public DataMapExpression dataMapExpressionTreeFormation () { + DataMapAndNodeExpression prevAndExpression = null; + DataMapExpression rootAndExpression = null; + DataMapColumnNodeExpression dataMapColumnNodeExpression = null; + List dataMapColumnNodeExpressionList = new ArrayList<>(); + // Form the Datamap expression tree based on coverage and dataMapFactoryListMap. + + /** + * The DataMap Tree formation is somewhat like This. + * + * DataMapNode + * / \ + * / \ + * FilterNode FilterNode + * ? \ / \ + * / \ / \ + * ColumnNode Literal ColumnNode Literal. + * + * + * AndNode + * / \ + * / \ + * / \ + * DataMapNode DataMapNode + * / \ + * / \ + * FilterNode FilterNode + * / \ / \ + * / \ / \ + * ColumnNode Literal ColumnNode Literal + */ + + // Major points. + // Multiple DataMaps can be linked with And Node. + // DataMapNode can have multiple children and can have FilterNode as a child or + // multiple filterNodes depending on the number of operators being linked to + // each DataMap. + if (coverage.equals(DataMapCoveringOptions.DataMapCovered.FULLY_COVERED)) { + // There will a single DataMap. + for (Map.Entry> dataMapFactoryListEntry : + dataMapFactoryListMap.entrySet()) { + List dataMapColumnExpressionList = + dataMapFactoryListEntry.getValue(); + List filterNodes = new ArrayList<>(dataMapColumnExpressionList.size()); + for (DataMapColumnExpression dataMapExpression1 : dataMapColumnExpressionList) { + if (dataMapExpression1.getFilterType().equals(EQUALTO)) { + DataMapFilterNode equalFilterNode = new DataMapEqualNodeExpression( + new DataMapColumnNameNodeExpression(dataMapExpression1.getColumnName()), null, + dataMapExpression1.getExpr(), dataMapFactoryListEntry.getKey(), + dataMapFactoryListEntry.getKey().getMeta()); + filterNodes.add(equalFilterNode); + } else if (dataMapExpression1.getFilterType().equals(GREATER_THAN)) { + DataMapFilterNode greaterThanFilterNode = new DataMapGreaterThanNodeExpression( + new DataMapColumnNameNodeExpression(dataMapExpression1.getColumnName()), null, + dataMapExpression1.getExpr(), dataMapFactoryListEntry.getKey(), + dataMapFactoryListEntry.getKey().getMeta()); + filterNodes.add(greaterThanFilterNode); + } else if (dataMapExpression1.getFilterType().equals(GREATER_THAN_EQUAL)) { + DataMapFilterNode greaterThanEqualFilterNode = + new DataMapGreaterThanEqualNodeExpression( + new DataMapColumnNameNodeExpression(dataMapExpression1.getColumnName()), null, + dataMapExpression1.getExpr(), dataMapFactoryListEntry.getKey(), + dataMapFactoryListEntry.getKey().getMeta()); + filterNodes.add(greaterThanEqualFilterNode); + } else if (dataMapExpression1.getFilterType().equals(LESS_THAN)) { + DataMapFilterNode lessThanFilterNode = new DataMapLessThanNodeExpression( + new DataMapColumnNameNodeExpression(dataMapExpression1.getColumnName()), null, + dataMapExpression1.getExpr(), dataMapFactoryListEntry.getKey(), + dataMapFactoryListEntry.getKey().getMeta()); + filterNodes.add(lessThanFilterNode); + } else if (dataMapExpression1.getFilterType().equals(LESS_THAN_EQUAL)) { + DataMapFilterNode lessThanEqualFilterNode = new DataMapLessThanEqualNodeExpression( + new DataMapColumnNameNodeExpression(dataMapExpression1.getColumnName()), null, + dataMapExpression1.getExpr(), dataMapFactoryListEntry.getKey(), + dataMapFactoryListEntry.getKey().getMeta()); + filterNodes.add(lessThanEqualFilterNode); + } else if (dataMapExpression1.getFilterType().equals(LIKE)) { + DataMapFilterNode likeFilterNode = new DataMapLikeNodeExpression( + new DataMapColumnNameNodeExpression(dataMapExpression1.getColumnName()), null, + dataMapExpression1.getExpr(), dataMapFactoryListEntry.getKey(), + dataMapFactoryListEntry.getKey().getMeta()); + filterNodes.add(likeFilterNode); + } + } + dataMapColumnNodeExpression = new DataMapColumnNodeExpression(filterNodes); + } + rootAndExpression = dataMapColumnNodeExpression; + return rootAndExpression; + } else if (coverage.equals(DataMapCoveringOptions.DataMapCovered.PARTIALLY_COVERED)) { + for (Map.Entry> dataMapFactoryListEntry : + dataMapFactoryListMap.entrySet()) { + List dataMapColumnExpressionList = + dataMapFactoryListEntry.getValue(); + List filterNodes = new ArrayList<>(dataMapColumnExpressionList.size()); + for (DataMapColumnExpression dataMapExpression1 : dataMapColumnExpressionList) { + if (dataMapExpression1.getFilterType().equals(EQUALTO)) { + DataMapFilterNode equalFilterNode = new DataMapEqualNodeExpression( + new DataMapColumnNameNodeExpression(dataMapExpression1.getColumnName()), null, + dataMapExpression1.getExpr(), dataMapFactoryListEntry.getKey(), + dataMapFactoryListEntry.getKey().getMeta()); + filterNodes.add(equalFilterNode); + } else if (dataMapExpression1.getFilterType().equals(GREATER_THAN)) { + DataMapFilterNode greaterThanFilterNode = new DataMapGreaterThanNodeExpression( + new DataMapColumnNameNodeExpression(dataMapExpression1.getColumnName()), null, + dataMapExpression1.getExpr(), dataMapFactoryListEntry.getKey(), + dataMapFactoryListEntry.getKey().getMeta()); + filterNodes.add(greaterThanFilterNode); + } else if (dataMapExpression1.getFilterType().equals(GREATER_THAN_EQUAL)) { + DataMapFilterNode greaterThanEqualFilterNode = + new DataMapGreaterThanEqualNodeExpression( + new DataMapColumnNameNodeExpression(dataMapExpression1.getColumnName()), null, + dataMapExpression1.getExpr(), dataMapFactoryListEntry.getKey(), + dataMapFactoryListEntry.getKey().getMeta()); + filterNodes.add(greaterThanEqualFilterNode); + } else if (dataMapExpression1.getFilterType().equals(LESS_THAN)) { + DataMapFilterNode lessThanFilterNode = new DataMapLessThanNodeExpression( + new DataMapColumnNameNodeExpression(dataMapExpression1.getColumnName()), null, + dataMapExpression1.getExpr(), dataMapFactoryListEntry.getKey(), + dataMapFactoryListEntry.getKey().getMeta()); + filterNodes.add(lessThanFilterNode); + } else if (dataMapExpression1.getFilterType().equals(LESS_THAN_EQUAL)) { + DataMapFilterNode lessThanEqualFilterNode = new DataMapLessThanEqualNodeExpression( + new DataMapColumnNameNodeExpression(dataMapExpression1.getColumnName()), null, + dataMapExpression1.getExpr(), dataMapFactoryListEntry.getKey(), + dataMapFactoryListEntry.getKey().getMeta()); + filterNodes.add(lessThanEqualFilterNode); + } else if (dataMapExpression1.getFilterType().equals(LIKE)) { + DataMapFilterNode likeFilterNode = new DataMapLikeNodeExpression( + new DataMapColumnNameNodeExpression(dataMapExpression1.getColumnName()), null, + dataMapExpression1.getExpr(), dataMapFactoryListEntry.getKey(), + dataMapFactoryListEntry.getKey().getMeta()); + filterNodes.add(likeFilterNode); + } + } + dataMapColumnNodeExpression = new DataMapColumnNodeExpression(filterNodes); + dataMapColumnNodeExpressionList.add(dataMapColumnNodeExpression); + } + + // For a Tree with AND + if (dataMapColumnNodeExpressionList.size() > 1) { + int index = 0; + List finalList = new ArrayList<>(); + for (index = 0; index < dataMapColumnNodeExpressionList.size() - 1; index++) { + finalList.add(new DataMapAndNodeExpression(null, null)); + finalList.add(dataMapColumnNodeExpressionList.get(index)); + } + while (index < dataMapColumnNodeExpressionList.size()) { + finalList.add(dataMapColumnNodeExpressionList.get(index)); + } + + // form the tree. + rootAndExpression = finalList.get(0); + for (index = 0; index < finalList.size(); index++) { + if (finalList.get(index) instanceof DataMapAndNodeExpression) { + DataMapAndNodeExpression andNodeExpression = + (DataMapAndNodeExpression) finalList.get(index); + andNodeExpression.setLeft(finalList.get(++index)); + andNodeExpression.setRight(finalList.get(++index)); + } + } + } else { + rootAndExpression = dataMapColumnNodeExpressionList.get(0); + } + return rootAndExpression; + } + return null; + } +// +// public TableDataMap extractDataMapsFromExpressionTree(DataMapExpression dataMapExpression) { +// TableDataMap tableDataMap = new TableDataMap(); +// +// } +} + diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapExpressionType.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapExpressionType.java new file mode 100644 index 00000000000..983df90415c --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapExpressionType.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.core.datamap.dev; + +public enum DataMapExpressionType { + + NODE_DATAMAP, + AND_DATAMAP, + OR_DATAMAP, + LITERAL_DATAMAP, + COLUMN_DATAMAP, + EQUALS_DATAMAP_FILTERTYPE, + GREATERTHAN_DATAMAP_FILTERTYPE, + GREATERTHAN_EQUALTO_DATAMAP_FILTERTYPE, + LESSTHAN_DATAMAP_FILTERTYPE, + LESSTHAN_EQUALTO_DATAMAP_FILTERTYPE, + LIKE_DATAMAP_FILTERTYPE +} diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFilterNode.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFilterNode.java new file mode 100644 index 00000000000..eb2775fb65c --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFilterNode.java @@ -0,0 +1,38 @@ +package org.apache.carbondata.core.datamap.dev; + +import org.apache.carbondata.core.datamap.DataMapMeta; +import org.apache.carbondata.core.scan.expression.Expression; + +public abstract class DataMapFilterNode extends DataMapExpression { + protected DataMapExpression columnExpression; + protected DataMapExpression literalExpression; + protected DataMapFactory dataMapFactory; + protected Expression filterExpression; + protected DataMapMeta dataMapMeta; + + public DataMapFilterNode(DataMapExpression columnExpr, DataMapExpression literal, + Expression expr, DataMapFactory dataMapFactory, DataMapMeta dataMapMeta) { + this.columnExpression = columnExpr; + this.literalExpression = literal; + this.filterExpression = expr; + this.dataMapFactory = dataMapFactory; + this.dataMapMeta = dataMapMeta; + } + + public DataMapExpression getColumnExpression () { + return columnExpression; + } + + public DataMapExpression getLiteralExpression() { + return literalExpression; + } + + public Expression getFilterExpression () { + return filterExpression; + } + + public DataMapFactory getDataMapFactory () { return dataMapFactory; } + + public DataMapMeta getDataMapMeta () { return dataMapMeta; } + +} diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapGreaterThanEqualNodeExpression.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapGreaterThanEqualNodeExpression.java new file mode 100644 index 00000000000..92034dae1c5 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapGreaterThanEqualNodeExpression.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.core.datamap.dev; + +import java.util.List; + +import org.apache.carbondata.core.datamap.DataMapMeta; +import org.apache.carbondata.core.indexstore.Blocklet; +import org.apache.carbondata.core.scan.expression.Expression; +import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; + +public class DataMapGreaterThanEqualNodeExpression extends DataMapFilterNode { + + public DataMapGreaterThanEqualNodeExpression (DataMapExpression columnExpr, DataMapExpression literal, + Expression expr, DataMapFactory dataMapFactory, DataMapMeta dataMapMeta) { + super(columnExpr, literal, expr, dataMapFactory, dataMapMeta); + } + + @Override public DataMapExpressionType getDataMapExpressionType() { + return DataMapExpressionType.GREATERTHAN_EQUALTO_DATAMAP_FILTERTYPE; + } + + @Override public List prune(FilterResolverIntf filterExp) { + return null; + } +} diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapGreaterThanNodeExpression.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapGreaterThanNodeExpression.java new file mode 100644 index 00000000000..716ec39eff6 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapGreaterThanNodeExpression.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.core.datamap.dev; + +import java.util.List; + +import org.apache.carbondata.core.datamap.DataMapMeta; +import org.apache.carbondata.core.indexstore.Blocklet; +import org.apache.carbondata.core.scan.expression.Expression; +import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; + +public class DataMapGreaterThanNodeExpression extends DataMapFilterNode { + + public DataMapGreaterThanNodeExpression (DataMapExpression columnExpr, DataMapExpression literal, + Expression expr, DataMapFactory dataMapFactory, DataMapMeta dataMapMeta) { + super(columnExpr, literal, expr, dataMapFactory, dataMapMeta); + } + + @Override public DataMapExpressionType getDataMapExpressionType() { + return DataMapExpressionType.GREATERTHAN_DATAMAP_FILTERTYPE; + } + + @Override public List prune(FilterResolverIntf filterExp) { + return null; + } + +} diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapLessThanEqualNodeExpression.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapLessThanEqualNodeExpression.java new file mode 100644 index 00000000000..2da1054942d --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapLessThanEqualNodeExpression.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.core.datamap.dev; + +import java.util.List; + +import org.apache.carbondata.core.datamap.DataMapMeta; +import org.apache.carbondata.core.indexstore.Blocklet; +import org.apache.carbondata.core.scan.expression.Expression; +import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; + +public class DataMapLessThanEqualNodeExpression extends DataMapFilterNode { + + public DataMapLessThanEqualNodeExpression (DataMapExpression columnExpr, DataMapExpression literal, + Expression expr, DataMapFactory dataMapFactory, DataMapMeta dataMapMeta) { + super(columnExpr, literal, expr, dataMapFactory, dataMapMeta); + } + + @Override public DataMapExpressionType getDataMapExpressionType() { + return DataMapExpressionType.LESSTHAN_EQUALTO_DATAMAP_FILTERTYPE; + } + + @Override public List prune(FilterResolverIntf filterExp) { + return null; + } + +} diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapLessThanNodeExpression.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapLessThanNodeExpression.java new file mode 100644 index 00000000000..98aaf0d654b --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapLessThanNodeExpression.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.core.datamap.dev; + +import java.util.List; + +import org.apache.carbondata.core.datamap.DataMapMeta; +import org.apache.carbondata.core.indexstore.Blocklet; +import org.apache.carbondata.core.scan.expression.Expression; +import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; + +public class DataMapLessThanNodeExpression extends DataMapFilterNode { + + public DataMapLessThanNodeExpression (DataMapExpression columnExpr, DataMapExpression literal, + Expression expr, DataMapFactory dataMapFactory, DataMapMeta dataMapMeta) { + super(columnExpr, literal, expr, dataMapFactory, dataMapMeta); + } + + @Override public DataMapExpressionType getDataMapExpressionType() { + return DataMapExpressionType.LESSTHAN_DATAMAP_FILTERTYPE; + } + + @Override public List prune(FilterResolverIntf filterExp) { + return null; + } + +} diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapLikeNodeExpression.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapLikeNodeExpression.java new file mode 100644 index 00000000000..cac0ef29295 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapLikeNodeExpression.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.core.datamap.dev; + +import java.util.List; + +import org.apache.carbondata.core.datamap.DataMapMeta; +import org.apache.carbondata.core.indexstore.Blocklet; +import org.apache.carbondata.core.scan.expression.Expression; +import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; + +public class DataMapLikeNodeExpression extends DataMapFilterNode { + + public DataMapLikeNodeExpression (DataMapExpression columnExpr, DataMapExpression literal, + Expression expr, DataMapFactory dataMapFactory, DataMapMeta dataMapMeta) { + super(columnExpr, literal, expr, dataMapFactory, dataMapMeta); + } + @Override public DataMapExpressionType getDataMapExpressionType() { + return DataMapExpressionType.LIKE_DATAMAP_FILTERTYPE; + } + + @Override public List prune(FilterResolverIntf filterExp) { + return null; + } + +} diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapLiteralNodeExpression.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapLiteralNodeExpression.java new file mode 100644 index 00000000000..7297e94dee8 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapLiteralNodeExpression.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.core.datamap.dev; + +import java.util.List; +import java.util.Objects; + +import org.apache.carbondata.core.indexstore.Blocklet; +import org.apache.carbondata.core.metadata.datatype.DataType; +import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; + +public class DataMapLiteralNodeExpression extends DataMapExpression { + protected Object value; + protected DataType dataType; + + public DataMapLiteralNodeExpression (Object value, DataType dataType) { + this.value = value; + this.dataType = dataType; + } + + public Object getValue () { + return value; + } + + public DataType getDataType () { + return dataType; + } + + @Override public DataMapExpressionType getDataMapExpressionType() { + return DataMapExpressionType.LITERAL_DATAMAP; + } + + @Override public List prune(FilterResolverIntf filterExp) { + return null; + } + +} diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapNodeExpression.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapNodeExpression.java new file mode 100644 index 00000000000..3e5ea77c7a4 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapNodeExpression.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.core.datamap.dev; + +import java.util.List; + +import org.apache.carbondata.core.indexstore.Blocklet; +import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; + +public class DataMapNodeExpression extends DataMapExpression { + + @Override public DataMapExpressionType getDataMapExpressionType() { + return DataMapExpressionType.NODE_DATAMAP; + } + + @Override public List prune(FilterResolverIntf filterExp) { + return null; + } + +} diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java index 314ffd5390a..cf01f9332aa 100644 --- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java +++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java @@ -272,7 +272,7 @@ private AbsoluteTableIdentifier getAbsoluteTableIdentifier(Configuration configu @Override public List getSplits(JobContext job) throws IOException { AbsoluteTableIdentifier identifier = getAbsoluteTableIdentifier(job.getConfiguration()); TableDataMap blockletMap = - DataMapStoreManager.getInstance().getDataMap(identifier, BlockletDataMap.NAME, + DataMapStoreManager.getInstance().getDataMap(identifier, null, BlockletDataMap.NAME, BlockletDataMapFactory.class.getName()); List invalidSegments = new ArrayList<>(); List invalidTimestampsList = new ArrayList<>(); @@ -536,7 +536,7 @@ private List getDataBlocksOfSegme new Path[] { new Path(absoluteTableIdentifier.getTablePath()) }, job.getConfiguration()); TableDataMap blockletMap = DataMapStoreManager.getInstance() - .getDataMap(absoluteTableIdentifier, BlockletDataMap.NAME, + .getDataMap(absoluteTableIdentifier, resolver, BlockletDataMap.NAME, BlockletDataMapFactory.class.getName()); DataMapJob dataMapJob = getDataMapJob(job.getConfiguration()); List prunedBlocklets; @@ -713,7 +713,7 @@ private String[] getSegmentsToAccess(JobContext job) { public BlockMappingVO getBlockRowCount(JobContext job, AbsoluteTableIdentifier identifier) throws IOException, KeyGenException { TableDataMap blockletMap = DataMapStoreManager.getInstance() - .getDataMap(identifier, BlockletDataMap.NAME, BlockletDataMapFactory.class.getName()); + .getDataMap(identifier, null, BlockletDataMap.NAME, BlockletDataMapFactory.class.getName()); SegmentUpdateStatusManager updateStatusManager = new SegmentUpdateStatusManager(identifier); SegmentStatusManager.ValidAndInvalidSegmentsInfo validAndInvalidSegments = new SegmentStatusManager(identifier).getValidAndInvalidSegments(); diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/DistributableDataMapFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/DistributableDataMapFormat.java index 66a06ba0367..dd9cd9885c2 100644 --- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/DistributableDataMapFormat.java +++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/DistributableDataMapFormat.java @@ -81,7 +81,7 @@ private static FilterResolverIntf getFilterExp(Configuration configuration) thro @Override public List getSplits(JobContext job) throws IOException { TableDataMap dataMap = - DataMapStoreManager.getInstance().getDataMap(identifier, dataMapName, className); + DataMapStoreManager.getInstance().getDataMap(identifier, null, dataMapName, className); List distributables = dataMap.toDistributable(validSegments); List inputSplits = new ArrayList<>(distributables.size()); inputSplits.addAll(distributables); @@ -102,7 +102,7 @@ public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptCont AbsoluteTableIdentifier identifier = AbsoluteTableIdentifier.fromTablePath(distributable.getTablePath()); TableDataMap dataMap = DataMapStoreManager.getInstance() - .getDataMap(identifier, distributable.getDataMapName(), + .getDataMap(identifier, null, distributable.getDataMapName(), distributable.getDataMapFactoryClass()); blockletIterator = dataMap.prune(distributable, getFilterExp(taskAttemptContext.getConfiguration())) diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala index ba6ad31dd92..af1443ee43a 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala @@ -20,6 +20,7 @@ package org.apache.carbondata.spark.testsuite.datamap import java.util import scala.collection.JavaConverters._ +import scala.collection.mutable import org.apache.spark.sql.{DataFrame, SaveMode} import org.apache.spark.sql.test.util.QueryTest @@ -51,7 +52,11 @@ class C2DataMapFactory() extends DataMapFactory { override def createWriter(segmentId: String): DataMapWriter = DataMapWriterSuite.dataMapWriterC2Mock - override def getMeta: DataMapMeta = new DataMapMeta(List("c2").asJava, FilterType.EQUALTO) + + var meta = new java.util.HashMap[String, FilterType]() + meta.put("c2", FilterType.EQUALTO) + + override def getMeta: DataMapMeta = new DataMapMeta(meta) /** * Get all distributable objects of a segmentid diff --git a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java index 4b0113c4566..7b8dbc24915 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java +++ b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java @@ -31,6 +31,7 @@ import org.apache.carbondata.core.datamap.dev.DataMapFactory; import org.apache.carbondata.core.datamap.dev.DataMapWriter; import org.apache.carbondata.core.datastore.page.ColumnPage; +import org.apache.carbondata.core.indexstore.schema.FilterType; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.processing.store.TablePage; @@ -69,7 +70,8 @@ private void register(DataMapFactory factory, String segmentId) { // if data map does not have meta, no need to register return; } - List columns = factory.getMeta().getIndexedColumns(); + Map columnsFilterType = factory.getMeta().getIndexedColumns(); + List columns = new ArrayList<>(columnsFilterType.keySet()); List writers = registry.get(columns); DataMapWriter writer = factory.createWriter(segmentId); if (writers != null) {