From 0358d65ede29f6982436fd060255ab4dbe1e0abc Mon Sep 17 00:00:00 2001 From: Vitalii Diravka Date: Wed, 17 Apr 2019 15:46:54 +0300 Subject: [PATCH] DRILL-7098: File Metadata Metastore Plugin --- exec/java-exec/pom.xml | 5 ++ .../apache/drill/exec/expr/IsPredicate.java | 4 +- .../base/AbstractGroupScanWithMetadata.java | 13 ++-- .../drill/exec/physical/base/GroupScan.java | 2 - .../base/SimpleFileTableMetadataProvider.java | 2 +- .../statistics/AbstractMergedStatistic.java | 2 +- .../logical/ConvertCountToDirectScanRule.java | 4 +- .../ConvertCountToDirectScanPrule.java | 3 +- .../record/metadata/MapColumnMetadata.java | 3 +- .../exec/record/metadata/MetadataUtils.java | 37 +++++++++ .../parquet/AbstractParquetGroupScan.java | 3 +- .../parquet/BaseParquetMetadataProvider.java | 11 +-- .../parquet/ParquetGroupScanStatistics.java | 8 +- .../parquet/ParquetTableMetadataUtils.java | 10 +-- .../parquet/metadata/MetadataPathUtils.java | 13 ++-- .../apache/drill/jdbc/impl/DrillMetaImpl.java | 17 ++--- metastore/file-metadata/pom.xml | 33 ++++++++ metastore/metastore-api/pom.xml | 33 ++++++++ .../exec/expr/ExactStatisticsConstants.java | 0 .../physical/impl/statistics/Statistic.java | 36 ++++----- .../rowSet/impl/SchemaTransformer.java | 0 .../exec/record/metadata/SchemaContainer.java | 0 .../exec/record/metadata/SchemaPathUtils.java | 34 --------- .../apache/drill/metastore/BaseMetadata.java | 0 .../CollectableColumnStatisticsKind.java | 0 .../CollectableTableStatisticsKind.java | 0 .../drill/metastore/ColumnStatistics.java | 0 .../drill/metastore/ColumnStatisticsImpl.java | 0 .../drill/metastore/ColumnStatisticsKind.java | 5 +- .../apache/drill/metastore/FileMetadata.java | 0 .../drill/metastore/FileTableMetadata.java | 0 .../drill/metastore/LocationProvider.java | 0 .../drill/metastore/PartitionMetadata.java | 0 .../drill/metastore/RowGroupMetadata.java | 0 .../drill/metastore/StatisticsKind.java | 0 .../apache/drill/metastore/TableMetadata.java | 0 .../drill/metastore/TableStatisticsKind.java | 7 +- metastore/pom.xml | 75 +++++++++++++++++++ pom.xml | 1 + 39 files changed, 253 insertions(+), 108 deletions(-) create mode 100644 metastore/file-metadata/pom.xml create mode 100644 metastore/metastore-api/pom.xml rename {exec/java-exec => metastore/metastore-api}/src/main/java/org/apache/drill/exec/expr/ExactStatisticsConstants.java (100%) rename {exec/java-exec => metastore/metastore-api}/src/main/java/org/apache/drill/exec/physical/impl/statistics/Statistic.java (54%) rename {exec/java-exec => metastore/metastore-api}/src/main/java/org/apache/drill/exec/physical/rowSet/impl/SchemaTransformer.java (100%) rename {exec/java-exec => metastore/metastore-api}/src/main/java/org/apache/drill/exec/record/metadata/SchemaContainer.java (100%) rename {exec/java-exec => metastore/metastore-api}/src/main/java/org/apache/drill/exec/record/metadata/SchemaPathUtils.java (55%) rename {exec/java-exec => metastore/metastore-api}/src/main/java/org/apache/drill/metastore/BaseMetadata.java (100%) rename {exec/java-exec => metastore/metastore-api}/src/main/java/org/apache/drill/metastore/CollectableColumnStatisticsKind.java (100%) rename {exec/java-exec => metastore/metastore-api}/src/main/java/org/apache/drill/metastore/CollectableTableStatisticsKind.java (100%) rename {exec/java-exec => metastore/metastore-api}/src/main/java/org/apache/drill/metastore/ColumnStatistics.java (100%) rename {exec/java-exec => metastore/metastore-api}/src/main/java/org/apache/drill/metastore/ColumnStatisticsImpl.java (100%) rename {exec/java-exec => metastore/metastore-api}/src/main/java/org/apache/drill/metastore/ColumnStatisticsKind.java (96%) rename {exec/java-exec => metastore/metastore-api}/src/main/java/org/apache/drill/metastore/FileMetadata.java (100%) rename {exec/java-exec => metastore/metastore-api}/src/main/java/org/apache/drill/metastore/FileTableMetadata.java (100%) rename {exec/java-exec => metastore/metastore-api}/src/main/java/org/apache/drill/metastore/LocationProvider.java (100%) rename {exec/java-exec => metastore/metastore-api}/src/main/java/org/apache/drill/metastore/PartitionMetadata.java (100%) rename {exec/java-exec => metastore/metastore-api}/src/main/java/org/apache/drill/metastore/RowGroupMetadata.java (100%) rename {exec/java-exec => metastore/metastore-api}/src/main/java/org/apache/drill/metastore/StatisticsKind.java (100%) rename {exec/java-exec => metastore/metastore-api}/src/main/java/org/apache/drill/metastore/TableMetadata.java (100%) rename {exec/java-exec => metastore/metastore-api}/src/main/java/org/apache/drill/metastore/TableStatisticsKind.java (93%) create mode 100644 metastore/pom.xml diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml index 1d0cb75193c..817185453dd 100644 --- a/exec/java-exec/pom.xml +++ b/exec/java-exec/pom.xml @@ -286,6 +286,11 @@ vector ${project.version} + + org.apache.drill.metastore + drill-metastore-api + ${project.version} + com.beust jcommander diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/IsPredicate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/IsPredicate.java index c30879db33b..a5ec6bfdab6 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/IsPredicate.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/IsPredicate.java @@ -18,7 +18,7 @@ package org.apache.drill.exec.expr; import org.apache.drill.exec.expr.stat.RowsMatch; -import org.apache.drill.exec.physical.base.GroupScan; +import org.apache.drill.exec.physical.impl.statistics.Statistic; import org.apache.drill.metastore.ColumnStatistics; import org.apache.drill.metastore.ColumnStatisticsKind; import org.apache.drill.shaded.guava.com.google.common.base.Preconditions; @@ -76,7 +76,7 @@ static boolean isNullOrEmpty(ColumnStatistics stat) { || !stat.containsStatistic(ColumnStatisticsKind.MIN_VALUE) || !stat.containsStatistic(ColumnStatisticsKind.MAX_VALUE) || !stat.containsStatistic(ColumnStatisticsKind.NULLS_COUNT) - || (long) stat.getStatistic(ColumnStatisticsKind.NULLS_COUNT) == GroupScan.NO_COLUMN_STATS; + || (long) stat.getStatistic(ColumnStatisticsKind.NULLS_COUNT) == Statistic.NO_COLUMN_STATS; } /** diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScanWithMetadata.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScanWithMetadata.java index 59d099fc03f..1b25582983f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScanWithMetadata.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScanWithMetadata.java @@ -33,6 +33,7 @@ import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.expr.stat.RowsMatch; import org.apache.drill.exec.ops.UdfUtilities; +import org.apache.drill.exec.physical.impl.statistics.Statistic; import org.apache.drill.exec.planner.physical.PlannerSettings; import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.record.metadata.ColumnMetadata; @@ -151,13 +152,13 @@ public long getColumnValueCount(SchemaPath column) { long colNulls; if (columnStats != null) { Long nulls = (Long) columnStats.getStatistic(ColumnStatisticsKind.NULLS_COUNT); - colNulls = nulls != null ? nulls : GroupScan.NO_COLUMN_STATS; + colNulls = nulls != null ? nulls : Statistic.NO_COLUMN_STATS; } else { return 0; } - return GroupScan.NO_COLUMN_STATS == tableRowCount - || GroupScan.NO_COLUMN_STATS == colNulls - ? GroupScan.NO_COLUMN_STATS : tableRowCount - colNulls; + return Statistic.NO_COLUMN_STATS == tableRowCount + || Statistic.NO_COLUMN_STATS == colNulls + ? Statistic.NO_COLUMN_STATS : tableRowCount - colNulls; } @Override @@ -339,7 +340,7 @@ public GroupScan applyLimit(int maxRecords) { GroupScanWithMetadataFilterer prunedMetadata = getFilterer(); if (getTableMetadata() != null) { long tableRowCount = (long) TableStatisticsKind.ROW_COUNT.getValue(getTableMetadata()); - if (tableRowCount == NO_COLUMN_STATS || tableRowCount <= maxRecords) { + if (tableRowCount == Statistic.NO_COLUMN_STATS || tableRowCount <= maxRecords) { logger.debug("limit push down does not apply, since total number of rows [{}] is less or equal to the required [{}].", tableRowCount, maxRecords); return null; @@ -404,7 +405,7 @@ protected List limitMetadata(Collection metadataL int currentRowCount = 0; for (T metadata : metadataList) { long rowCount = (long) TableStatisticsKind.ROW_COUNT.getValue(metadata); - if (rowCount == NO_COLUMN_STATS) { + if (rowCount == Statistic.NO_COLUMN_STATS) { return null; } else if (currentRowCount + rowCount <= maxRecords) { currentRowCount += rowCount; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java index ce74da1cd6c..cd49a1ea1ec 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java @@ -46,8 +46,6 @@ public interface GroupScan extends Scan, HasAffinity { */ List ALL_COLUMNS = ImmutableList.of(SchemaPath.STAR_COLUMN); - long NO_COLUMN_STATS = -1; - void applyAssignments(List endpoints) throws PhysicalOperatorSetupException; SubScan getSpecificScan(int minorFragmentId) throws ExecutionSetupException; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SimpleFileTableMetadataProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SimpleFileTableMetadataProvider.java index 98c0c197742..1efad36af8f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SimpleFileTableMetadataProvider.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SimpleFileTableMetadataProvider.java @@ -124,7 +124,7 @@ public SimpleFileTableMetadataProviderBuilder withSchema(TupleMetadata schema) { @Override @SuppressWarnings("unchecked") - public TableMetadataProvider build() throws IOException { + public TableMetadataProvider build() { SchemaProvider schemaProvider = metadataProviderManager.getSchemaProvider(); TableMetadataProvider source = metadataProviderManager.getTableMetadataProvider(); if (source == null) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/AbstractMergedStatistic.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/AbstractMergedStatistic.java index a4ee74fa450..4e9a7623907 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/AbstractMergedStatistic.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/AbstractMergedStatistic.java @@ -19,7 +19,7 @@ import org.apache.drill.exec.vector.complex.MapVector; -public abstract class AbstractMergedStatistic extends Statistic implements MergedStatistic { +public abstract class AbstractMergedStatistic implements MergedStatistic, Statistic { protected String name; protected String inputName; protected double samplePercent; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/ConvertCountToDirectScanRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/ConvertCountToDirectScanRule.java index fd84e20eee4..73754998daf 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/ConvertCountToDirectScanRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/ConvertCountToDirectScanRule.java @@ -31,8 +31,8 @@ import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.logical.FormatPluginConfig; -import org.apache.drill.exec.physical.base.GroupScan; import org.apache.drill.exec.physical.base.ScanStats; +import org.apache.drill.exec.physical.impl.statistics.Statistic; import org.apache.drill.exec.planner.common.CountToDirectScanUtils; import org.apache.drill.exec.planner.common.DrillRelOptUtil; @@ -288,7 +288,7 @@ private Map collectCounts(PlannerSettings settings, Metadata_V4.Me Metadata_V4.ColumnTypeMetadata_v4 columnMetadata = metadataSummary.getColumnTypeInfo(new Metadata_V4.ColumnTypeMetadata_v4.Key(simplePath)); - if (columnMetadata == null || columnMetadata.totalNullCount == GroupScan.NO_COLUMN_STATS) { + if (columnMetadata == null || columnMetadata.totalNullCount == Statistic.NO_COLUMN_STATS) { // if column stats is not available don't apply this rule, return empty counts return ImmutableMap.of(); } else { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ConvertCountToDirectScanPrule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ConvertCountToDirectScanPrule.java index 41769506084..8d7022eba3c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ConvertCountToDirectScanPrule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ConvertCountToDirectScanPrule.java @@ -33,6 +33,7 @@ import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.exec.physical.base.GroupScan; import org.apache.drill.exec.physical.base.ScanStats; +import org.apache.drill.exec.physical.impl.statistics.Statistic; import org.apache.drill.exec.planner.logical.DrillAggregateRel; import org.apache.drill.exec.planner.logical.DrillProjectRel; import org.apache.drill.exec.planner.logical.DrillScanRel; @@ -202,7 +203,7 @@ private Map collectCounts(PlannerSettings settings, DrillAggregate } cnt = oldGrpScan.getColumnValueCount(simplePath); - if (cnt == GroupScan.NO_COLUMN_STATS) { + if (cnt == Statistic.NO_COLUMN_STATS) { // if column stats is not available don't apply this rule, return empty counts return ImmutableMap.of(); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/MapColumnMetadata.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/MapColumnMetadata.java index bf1f9afd6b7..f9c05f4e3fa 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/MapColumnMetadata.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/MapColumnMetadata.java @@ -64,8 +64,7 @@ public MapColumnMetadata(MapColumnMetadata from) { mapSchema = (TupleSchema) from.mapSchema.copy(); } - public MapColumnMetadata(String name, DataMode mode, - TupleSchema mapSchema) { + public MapColumnMetadata(String name, DataMode mode, TupleSchema mapSchema) { super(name, MinorType.MAP, mode); if (mapSchema == null) { this.mapSchema = new TupleSchema(); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/MetadataUtils.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/MetadataUtils.java index 21577c0c87f..f1cf45a30e8 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/MetadataUtils.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/MetadataUtils.java @@ -19,6 +19,10 @@ import java.util.List; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.expression.PathSegment; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; @@ -203,4 +207,37 @@ private static ColumnMetadata newDecimal(String name, MinorType type, DataMode m .build(); return new PrimitiveColumnMetadata(field); } + + /** + * Adds column with specified schema path and type into specified {@code TupleMetadata schema}. + * + * @param schema tuple schema where column should be added + * @param schemaPath schema path of the column which should be added + * @param type type of the column which should be added + */ + public static void addColumnMetadata(TupleMetadata schema, SchemaPath schemaPath, TypeProtos.MajorType type) { + PathSegment.NameSegment colPath = schemaPath.getUnIndexed().getRootSegment(); + ColumnMetadata colMetadata; + + while (!colPath.isLastPath()) { + colMetadata = schema.metadata(colPath.getPath()); + if (colMetadata == null) { + colMetadata = MetadataUtils.newMap(colPath.getPath(), null); + schema.addColumn(colMetadata); + } + if (!colMetadata.isMap()) { + throw new DrillRuntimeException(String.format("Expected map, but was %s", colMetadata.majorType())); + } + + schema = colMetadata.mapSchema(); + colPath = (PathSegment.NameSegment) colPath.getChild(); + } + + colMetadata = schema.metadata(colPath.getPath()); + if (colMetadata == null) { + schema.addColumn(new PrimitiveColumnMetadata(MaterializedField.create(colPath.getPath(), type))); + } else if (!colMetadata.majorType().equals(type)) { + throw new DrillRuntimeException(String.format("Types mismatch: existing type: %s, new type: %s", colMetadata.majorType(), type)); + } + } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/AbstractParquetGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/AbstractParquetGroupScan.java index 5986b084672..03cdcfba393 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/AbstractParquetGroupScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/AbstractParquetGroupScan.java @@ -24,6 +24,7 @@ import org.apache.drill.common.expression.ExpressionStringBuilder; import org.apache.drill.exec.physical.base.AbstractGroupScanWithMetadata; import org.apache.drill.exec.physical.base.ParquetMetadataProvider; +import org.apache.drill.exec.physical.impl.statistics.Statistic; import org.apache.drill.metastore.BaseMetadata; import org.apache.drill.metastore.LocationProvider; import org.apache.drill.metastore.PartitionMetadata; @@ -319,7 +320,7 @@ public GroupScan applyLimit(int maxRecords) { maxRecords = Math.max(maxRecords, 1); // Make sure it request at least 1 row -> 1 rowGroup. if (getTableMetadata() != null) { long tableRowCount = (long) TableStatisticsKind.ROW_COUNT.getValue(getTableMetadata()); - if (tableRowCount == NO_COLUMN_STATS || tableRowCount <= maxRecords) { + if (tableRowCount == Statistic.NO_COLUMN_STATS || tableRowCount <= maxRecords) { logger.debug("limit push down does not apply, since total number of rows [{}] is less or equal to the required [{}].", tableRowCount, maxRecords); return null; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/BaseParquetMetadataProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/BaseParquetMetadataProvider.java index 29126b3ae07..5fac9834bc2 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/BaseParquetMetadataProvider.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/BaseParquetMetadataProvider.java @@ -17,9 +17,10 @@ */ package org.apache.drill.exec.store.parquet; -import org.apache.drill.exec.physical.base.GroupScan; import org.apache.drill.exec.physical.base.ParquetMetadataProvider; +import org.apache.drill.exec.physical.impl.statistics.Statistic; import org.apache.drill.exec.planner.common.DrillStatsTable; +import org.apache.drill.exec.record.metadata.MetadataUtils; import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.metastore.BaseMetadata; import org.apache.drill.metastore.ColumnStatisticsImpl; @@ -190,12 +191,12 @@ public TableMetadata getTableMetadata() { if (this.schema == null) { schema = new TupleSchema(); - fields.forEach((schemaPath, majorType) -> SchemaPathUtils.addColumnMetadata(schema, schemaPath, majorType)); + fields.forEach((schemaPath, majorType) -> MetadataUtils.addColumnMetadata(schema, schemaPath, majorType)); } else { // merges specified schema with schema from table fields.forEach((schemaPath, majorType) -> { if (SchemaPathUtils.getColumnMetadata(schemaPath, schema) == null) { - SchemaPathUtils.addColumnMetadata(schema, schemaPath, majorType); + MetadataUtils.addColumnMetadata(schema, schemaPath, majorType); } }); } @@ -306,8 +307,8 @@ public List getPartitionsMetadata() { statistics.put(ColumnStatisticsKind.MIN_VALUE, partitionKey); statistics.put(ColumnStatisticsKind.MAX_VALUE, partitionKey); - statistics.put(ColumnStatisticsKind.NULLS_COUNT, GroupScan.NO_COLUMN_STATS); - statistics.put(TableStatisticsKind.ROW_COUNT, GroupScan.NO_COLUMN_STATS); + statistics.put(ColumnStatisticsKind.NULLS_COUNT, Statistic.NO_COLUMN_STATS); + statistics.put(TableStatisticsKind.ROW_COUNT, Statistic.NO_COLUMN_STATS); columnsStatistics.put(partitionColumn, new ColumnStatisticsImpl<>(statistics, ParquetTableMetadataUtils.getComparator(getParquetGroupScanStatistics().getTypeForColumn(partitionColumn).getMinorType()))); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScanStatistics.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScanStatistics.java index d53c038bd77..c53cee9b8bb 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScanStatistics.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScanStatistics.java @@ -20,7 +20,7 @@ import org.apache.commons.lang3.mutable.MutableLong; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.types.TypeProtos; -import org.apache.drill.exec.physical.base.GroupScan; +import org.apache.drill.exec.physical.impl.statistics.Statistic; import org.apache.drill.exec.record.metadata.ColumnMetadata; import org.apache.drill.exec.record.metadata.SchemaPathUtils; import org.apache.drill.metastore.BaseMetadata; @@ -109,10 +109,10 @@ public void collect(List metadataList) { previousCount = emptyCount; } Long nullsNum = (Long) statistics.getStatistic(ColumnStatisticsKind.NULLS_COUNT); - if (previousCount.longValue() != GroupScan.NO_COLUMN_STATS && nullsNum != null && nullsNum != GroupScan.NO_COLUMN_STATS) { + if (previousCount.longValue() != Statistic.NO_COLUMN_STATS && nullsNum != null && nullsNum != Statistic.NO_COLUMN_STATS) { previousCount.add(localRowCount - nullsNum); } else { - previousCount.setValue(GroupScan.NO_COLUMN_STATS); + previousCount.setValue(Statistic.NO_COLUMN_STATS); } ColumnMetadata columnMetadata = SchemaPathUtils.getColumnMetadata(schemaPath, metadata.getSchema()); TypeProtos.MajorType majorType = columnMetadata != null ? columnMetadata.majorType() : null; @@ -207,7 +207,7 @@ private boolean hasSingleValue(ColumnStatistics columnStatistics, long rowCount) private boolean isSingleVal(ColumnStatistics columnStatistics, long rowCount) { Long numNulls = (Long) columnStatistics.getStatistic(ColumnStatisticsKind.NULLS_COUNT); - if (numNulls != null && numNulls != GroupScan.NO_COLUMN_STATS) { + if (numNulls != null && numNulls != Statistic.NO_COLUMN_STATS) { Object min = columnStatistics.getStatistic(ColumnStatisticsKind.MIN_VALUE); Object max = columnStatistics.getStatistic(ColumnStatisticsKind.MAX_VALUE); if (min != null) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetTableMetadataUtils.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetTableMetadataUtils.java index b1a686b1f59..9ec096bec32 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetTableMetadataUtils.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetTableMetadataUtils.java @@ -19,8 +19,8 @@ import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.types.TypeProtos; -import org.apache.drill.exec.physical.base.GroupScan; -import org.apache.drill.exec.record.metadata.SchemaPathUtils; +import org.apache.drill.exec.physical.impl.statistics.Statistic; +import org.apache.drill.exec.record.metadata.MetadataUtils; import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.record.metadata.TupleSchema; import org.apache.drill.exec.resolver.TypeCastRules; @@ -154,7 +154,7 @@ public static RowGroupMetadata getRowGroupMetadata(MetadataBase.ParquetTableMeta Map columns = getRowGroupFields(tableMetadata, rowGroupMetadata); TupleSchema schema = new TupleSchema(); - columns.forEach((schemaPath, majorType) -> SchemaPathUtils.addColumnMetadata(schema, schemaPath, majorType)); + columns.forEach((schemaPath, majorType) -> MetadataUtils.addColumnMetadata(schema, schemaPath, majorType)); return new RowGroupMetadata( schema, columnsStatistics, rowGroupStatistics, rowGroupMetadata.getHostAffinity(), rgIndexInFile, location); @@ -274,7 +274,7 @@ private static Map getRowGroupColumnStatistics( Long nulls = column.getNulls(); if (!column.isNumNullsSet() || nulls == null) { - nulls = GroupScan.NO_COLUMN_STATS; + nulls = Statistic.NO_COLUMN_STATS; } PrimitiveType.PrimitiveTypeName primitiveType = getPrimitiveTypeName(tableMetadata, column); OriginalType originalType = getOriginalType(tableMetadata, column); @@ -306,7 +306,7 @@ public static Map populateNonInterestingColumnsSta SchemaPath schemaPath = SchemaPath.getCompoundPath(columnTypeMetadata.name); if (!schemaPaths.contains(schemaPath)) { Map statistics = new HashMap<>(); - statistics.put(ColumnStatisticsKind.NULLS_COUNT, GroupScan.NO_COLUMN_STATS); + statistics.put(ColumnStatisticsKind.NULLS_COUNT, Statistic.NO_COLUMN_STATS); PrimitiveType.PrimitiveTypeName primitiveType = columnTypeMetadata.primitiveType; OriginalType originalType = columnTypeMetadata.originalType; Comparator comparator = getComparator(primitiveType, originalType); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/MetadataPathUtils.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/MetadataPathUtils.java index b94f135ec79..2cad8e1681e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/MetadataPathUtils.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/MetadataPathUtils.java @@ -17,7 +17,6 @@ */ package org.apache.drill.exec.store.parquet.metadata; -import org.apache.drill.shaded.guava.com.google.common.collect.Lists; import org.apache.drill.common.util.DrillVersionInfo; import org.apache.hadoop.fs.Path; @@ -44,10 +43,9 @@ public class MetadataPathUtils { */ public static List convertToAbsolutePaths(List paths, String baseDir) { if (!paths.isEmpty()) { - List absolutePaths = Lists.newArrayList(); + List absolutePaths = new ArrayList<>(); for (Path relativePath : paths) { - Path absolutePath = (relativePath.isAbsolute()) ? relativePath - : new Path(baseDir, relativePath); + Path absolutePath = (relativePath.isAbsolute()) ? relativePath : new Path(baseDir, relativePath); absolutePaths.add(absolutePath); } return absolutePaths; @@ -62,9 +60,10 @@ public static List convertToAbsolutePaths(List paths, String baseDir * @param baseDir base parent directory * @return list of files with absolute paths */ - public static List convertToFilesWithAbsolutePaths(List files, String baseDir) { + public static List convertToFilesWithAbsolutePaths( + List files, String baseDir) { if (!files.isEmpty()) { - List filesWithAbsolutePaths = Lists.newArrayList(); + List filesWithAbsolutePaths = new ArrayList<>(); for (ParquetFileMetadata file : files) { Path relativePath = file.getPath(); ParquetFileMetadata fileWithAbsolutePath = null; @@ -97,7 +96,7 @@ public static ParquetTableMetadata_v4 createMetadataWithRelativePaths( for (Path directory : tableMetadataWithAbsolutePaths.getDirectories()) { directoriesWithRelativePaths.add(relativize(baseDir, directory)); } - List filesWithRelativePaths = Lists.newArrayList(); + List filesWithRelativePaths = new ArrayList<>(); for (ParquetFileMetadata_v4 file : (List) tableMetadataWithAbsolutePaths.getFiles()) { filesWithRelativePaths.add(new ParquetFileMetadata_v4( relativize(baseDir, file.getPath()), file.length, file.rowGroups)); diff --git a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillMetaImpl.java b/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillMetaImpl.java index 2b61a6a3a41..e06445da9ce 100644 --- a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillMetaImpl.java +++ b/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillMetaImpl.java @@ -29,6 +29,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import javax.validation.constraints.NotNull; @@ -60,9 +61,7 @@ import org.apache.drill.exec.rpc.DrillRpcFuture; import org.apache.drill.exec.rpc.RpcException; -import org.apache.drill.shaded.guava.com.google.common.base.Function; import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableMap; -import org.apache.drill.shaded.guava.com.google.common.collect.Lists; public class DrillMetaImpl extends MetaImpl { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillMetaImpl.class); @@ -315,18 +314,14 @@ MetaResultSet getMeta(DrillRpcFuture future) { } try { - List tables = Lists.transform(getResult(response), new Function() { - @Override - public Object apply(ResponseValue input) { - return adapt(input); - } - }); + List tables = getResult(response).stream() + .map(this::adapt) + .collect(Collectors.toList()); Meta.Frame frame = Meta.Frame.create(0, true, tables); StructType fieldMetaData = drillFieldMetaData(clazz); - Meta.Signature signature = Meta.Signature.create( - fieldMetaData.columns, "", - Collections.emptyList(), CursorFactory.record(clazz), Meta.StatementType.SELECT); + Meta.Signature signature = Meta.Signature.create(fieldMetaData.columns, "", Collections.emptyList(), + CursorFactory.record(clazz), Meta.StatementType.SELECT); AvaticaStatement statement = connection.createStatement(); return MetaResultSet.create(connection.id, statement.getId(), true, diff --git a/metastore/file-metadata/pom.xml b/metastore/file-metadata/pom.xml new file mode 100644 index 00000000000..5d73c18ccf6 --- /dev/null +++ b/metastore/file-metadata/pom.xml @@ -0,0 +1,33 @@ + + + + 4.0.0 + + + org.apache.drill.metastore + metastore-parent + 1.16.0-SNAPSHOT + + + drill-file-metastore-plugin + Drill File Metadata Metastore Plugin + + diff --git a/metastore/metastore-api/pom.xml b/metastore/metastore-api/pom.xml new file mode 100644 index 00000000000..94511fb2358 --- /dev/null +++ b/metastore/metastore-api/pom.xml @@ -0,0 +1,33 @@ + + + + 4.0.0 + + + org.apache.drill.metastore + metastore-parent + 1.16.0-SNAPSHOT + + + drill-metastore-api + Drill Metastore API + + diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExactStatisticsConstants.java b/metastore/metastore-api/src/main/java/org/apache/drill/exec/expr/ExactStatisticsConstants.java similarity index 100% rename from exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExactStatisticsConstants.java rename to metastore/metastore-api/src/main/java/org/apache/drill/exec/expr/ExactStatisticsConstants.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/Statistic.java b/metastore/metastore-api/src/main/java/org/apache/drill/exec/physical/impl/statistics/Statistic.java similarity index 54% rename from exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/Statistic.java rename to metastore/metastore-api/src/main/java/org/apache/drill/exec/physical/impl/statistics/Statistic.java index 8f5392aaecc..5794a13185f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/Statistic.java +++ b/metastore/metastore-api/src/main/java/org/apache/drill/exec/physical/impl/statistics/Statistic.java @@ -20,27 +20,29 @@ /* * Base Statistics class - all statistics classes should extend this class */ -public abstract class Statistic { +public interface Statistic { /* * The lifecycle states for statistics */ - public enum State {INIT, CONFIG, MERGE, COMPLETE}; + enum State {INIT, CONFIG, MERGE, COMPLETE} + + long NO_COLUMN_STATS = -1; /* * List of statistics used in Drill. */ - public static final String COLNAME = "column"; - public static final String COLTYPE = "majortype"; - public static final String SCHEMA = "schema"; - public static final String COMPUTED = "computed"; - public static final String ROWCOUNT = "rowcount"; - public static final String NNROWCOUNT = "nonnullrowcount"; - public static final String NDV = "approx_count_distinct"; - public static final String HLL_MERGE = "hll_merge"; - public static final String HLL = "hll"; - public static final String AVG_WIDTH = "avg_width"; - public static final String SUM_WIDTH = "sum_width"; - public static final String CNT_DUPS = "approx_count_dups"; - public static final String SUM_DUPS = "sum"; - public static final String TDIGEST = "tdigest"; - public static final String TDIGEST_MERGE = "tdigest_merge"; + String COLNAME = "column"; + String COLTYPE = "majortype"; + String SCHEMA = "schema"; + String COMPUTED = "computed"; + String ROWCOUNT = "rowcount"; + String NNROWCOUNT = "nonnullrowcount"; + String NDV = "approx_count_distinct"; + String HLL_MERGE = "hll_merge"; + String HLL = "hll"; + String AVG_WIDTH = "avg_width"; + String SUM_WIDTH = "sum_width"; + String CNT_DUPS = "approx_count_dups"; + String SUM_DUPS = "sum"; + String TDIGEST = "tdigest"; + String TDIGEST_MERGE = "tdigest_merge"; } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/SchemaTransformer.java b/metastore/metastore-api/src/main/java/org/apache/drill/exec/physical/rowSet/impl/SchemaTransformer.java similarity index 100% rename from exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/SchemaTransformer.java rename to metastore/metastore-api/src/main/java/org/apache/drill/exec/physical/rowSet/impl/SchemaTransformer.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/SchemaContainer.java b/metastore/metastore-api/src/main/java/org/apache/drill/exec/record/metadata/SchemaContainer.java similarity index 100% rename from exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/SchemaContainer.java rename to metastore/metastore-api/src/main/java/org/apache/drill/exec/record/metadata/SchemaContainer.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/SchemaPathUtils.java b/metastore/metastore-api/src/main/java/org/apache/drill/exec/record/metadata/SchemaPathUtils.java similarity index 55% rename from exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/SchemaPathUtils.java rename to metastore/metastore-api/src/main/java/org/apache/drill/exec/record/metadata/SchemaPathUtils.java index 085c68c757f..5c39701b1da 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/SchemaPathUtils.java +++ b/metastore/metastore-api/src/main/java/org/apache/drill/exec/record/metadata/SchemaPathUtils.java @@ -17,11 +17,8 @@ */ package org.apache.drill.exec.record.metadata; -import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.expression.PathSegment; import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.common.types.TypeProtos; -import org.apache.drill.exec.record.MaterializedField; public class SchemaPathUtils { @@ -50,36 +47,5 @@ public static ColumnMetadata getColumnMetadata(SchemaPath schemaPath, TupleMetad return colMetadata; } - /** - * Adds column with specified schema path and type into specified {@code TupleMetadata schema}. - * - * @param schema tuple schema where column should be added - * @param schemaPath schema path of the column which should be added - * @param type type of the column which should be added - */ - public static void addColumnMetadata(TupleMetadata schema, SchemaPath schemaPath, TypeProtos.MajorType type) { - PathSegment.NameSegment colPath = schemaPath.getUnIndexed().getRootSegment(); - ColumnMetadata colMetadata; - while (!colPath.isLastPath()) { - colMetadata = schema.metadata(colPath.getPath()); - if (colMetadata == null) { - colMetadata = MetadataUtils.newMap(colPath.getPath(), null); - schema.addColumn(colMetadata); - } - if (!colMetadata.isMap()) { - throw new DrillRuntimeException(String.format("Expected map, but was %s", colMetadata.majorType())); - } - - schema = colMetadata.mapSchema(); - colPath = (PathSegment.NameSegment) colPath.getChild(); - } - - colMetadata = schema.metadata(colPath.getPath()); - if (colMetadata == null) { - schema.addColumn(new PrimitiveColumnMetadata(MaterializedField.create(colPath.getPath(), type))); - } else if (!colMetadata.majorType().equals(type)) { - throw new DrillRuntimeException(String.format("Types mismatch: existing type: %s, new type: %s", colMetadata.majorType(), type)); - } - } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/metastore/BaseMetadata.java b/metastore/metastore-api/src/main/java/org/apache/drill/metastore/BaseMetadata.java similarity index 100% rename from exec/java-exec/src/main/java/org/apache/drill/metastore/BaseMetadata.java rename to metastore/metastore-api/src/main/java/org/apache/drill/metastore/BaseMetadata.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/metastore/CollectableColumnStatisticsKind.java b/metastore/metastore-api/src/main/java/org/apache/drill/metastore/CollectableColumnStatisticsKind.java similarity index 100% rename from exec/java-exec/src/main/java/org/apache/drill/metastore/CollectableColumnStatisticsKind.java rename to metastore/metastore-api/src/main/java/org/apache/drill/metastore/CollectableColumnStatisticsKind.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/metastore/CollectableTableStatisticsKind.java b/metastore/metastore-api/src/main/java/org/apache/drill/metastore/CollectableTableStatisticsKind.java similarity index 100% rename from exec/java-exec/src/main/java/org/apache/drill/metastore/CollectableTableStatisticsKind.java rename to metastore/metastore-api/src/main/java/org/apache/drill/metastore/CollectableTableStatisticsKind.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/metastore/ColumnStatistics.java b/metastore/metastore-api/src/main/java/org/apache/drill/metastore/ColumnStatistics.java similarity index 100% rename from exec/java-exec/src/main/java/org/apache/drill/metastore/ColumnStatistics.java rename to metastore/metastore-api/src/main/java/org/apache/drill/metastore/ColumnStatistics.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/metastore/ColumnStatisticsImpl.java b/metastore/metastore-api/src/main/java/org/apache/drill/metastore/ColumnStatisticsImpl.java similarity index 100% rename from exec/java-exec/src/main/java/org/apache/drill/metastore/ColumnStatisticsImpl.java rename to metastore/metastore-api/src/main/java/org/apache/drill/metastore/ColumnStatisticsImpl.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/metastore/ColumnStatisticsKind.java b/metastore/metastore-api/src/main/java/org/apache/drill/metastore/ColumnStatisticsKind.java similarity index 96% rename from exec/java-exec/src/main/java/org/apache/drill/metastore/ColumnStatisticsKind.java rename to metastore/metastore-api/src/main/java/org/apache/drill/metastore/ColumnStatisticsKind.java index e94b8429f9c..da5bb1342be 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/metastore/ColumnStatisticsKind.java +++ b/metastore/metastore-api/src/main/java/org/apache/drill/metastore/ColumnStatisticsKind.java @@ -17,7 +17,6 @@ */ package org.apache.drill.metastore; -import org.apache.drill.exec.physical.base.GroupScan; import org.apache.drill.exec.expr.ExactStatisticsConstants; import org.apache.drill.exec.physical.impl.statistics.Statistic; @@ -38,8 +37,8 @@ public Object mergeStatistics(List statisticsList) { long nullsCount = 0; for (ColumnStatistics statistics : statisticsList) { Long statNullsCount = (Long) statistics.getStatistic(this); - if (statNullsCount == null || statNullsCount == GroupScan.NO_COLUMN_STATS) { - return GroupScan.NO_COLUMN_STATS; + if (statNullsCount == null || statNullsCount == Statistic.NO_COLUMN_STATS) { + return Statistic.NO_COLUMN_STATS; } else { nullsCount += statNullsCount; } diff --git a/exec/java-exec/src/main/java/org/apache/drill/metastore/FileMetadata.java b/metastore/metastore-api/src/main/java/org/apache/drill/metastore/FileMetadata.java similarity index 100% rename from exec/java-exec/src/main/java/org/apache/drill/metastore/FileMetadata.java rename to metastore/metastore-api/src/main/java/org/apache/drill/metastore/FileMetadata.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/metastore/FileTableMetadata.java b/metastore/metastore-api/src/main/java/org/apache/drill/metastore/FileTableMetadata.java similarity index 100% rename from exec/java-exec/src/main/java/org/apache/drill/metastore/FileTableMetadata.java rename to metastore/metastore-api/src/main/java/org/apache/drill/metastore/FileTableMetadata.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/metastore/LocationProvider.java b/metastore/metastore-api/src/main/java/org/apache/drill/metastore/LocationProvider.java similarity index 100% rename from exec/java-exec/src/main/java/org/apache/drill/metastore/LocationProvider.java rename to metastore/metastore-api/src/main/java/org/apache/drill/metastore/LocationProvider.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/metastore/PartitionMetadata.java b/metastore/metastore-api/src/main/java/org/apache/drill/metastore/PartitionMetadata.java similarity index 100% rename from exec/java-exec/src/main/java/org/apache/drill/metastore/PartitionMetadata.java rename to metastore/metastore-api/src/main/java/org/apache/drill/metastore/PartitionMetadata.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/metastore/RowGroupMetadata.java b/metastore/metastore-api/src/main/java/org/apache/drill/metastore/RowGroupMetadata.java similarity index 100% rename from exec/java-exec/src/main/java/org/apache/drill/metastore/RowGroupMetadata.java rename to metastore/metastore-api/src/main/java/org/apache/drill/metastore/RowGroupMetadata.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/metastore/StatisticsKind.java b/metastore/metastore-api/src/main/java/org/apache/drill/metastore/StatisticsKind.java similarity index 100% rename from exec/java-exec/src/main/java/org/apache/drill/metastore/StatisticsKind.java rename to metastore/metastore-api/src/main/java/org/apache/drill/metastore/StatisticsKind.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/metastore/TableMetadata.java b/metastore/metastore-api/src/main/java/org/apache/drill/metastore/TableMetadata.java similarity index 100% rename from exec/java-exec/src/main/java/org/apache/drill/metastore/TableMetadata.java rename to metastore/metastore-api/src/main/java/org/apache/drill/metastore/TableMetadata.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/metastore/TableStatisticsKind.java b/metastore/metastore-api/src/main/java/org/apache/drill/metastore/TableStatisticsKind.java similarity index 93% rename from exec/java-exec/src/main/java/org/apache/drill/metastore/TableStatisticsKind.java rename to metastore/metastore-api/src/main/java/org/apache/drill/metastore/TableStatisticsKind.java index 63b9243cf9b..48caa61683d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/metastore/TableStatisticsKind.java +++ b/metastore/metastore-api/src/main/java/org/apache/drill/metastore/TableStatisticsKind.java @@ -17,7 +17,6 @@ */ package org.apache.drill.metastore; -import org.apache.drill.exec.physical.base.GroupScan; import org.apache.drill.exec.expr.ExactStatisticsConstants; import org.apache.drill.exec.physical.impl.statistics.Statistic; @@ -37,8 +36,8 @@ public Long mergeStatistics(Collection statistics) { long rowCount = 0; for (BaseMetadata statistic : statistics) { Long statRowCount = getValue(statistic); - if (statRowCount == null || statRowCount == GroupScan.NO_COLUMN_STATS) { - rowCount = GroupScan.NO_COLUMN_STATS; + if (statRowCount == null || statRowCount == Statistic.NO_COLUMN_STATS) { + rowCount = Statistic.NO_COLUMN_STATS; break; } else { rowCount += statRowCount; @@ -50,7 +49,7 @@ public Long mergeStatistics(Collection statistics) { @Override public Long getValue(BaseMetadata metadata) { Long rowCount = (Long) metadata.getStatistic(this); - return rowCount != null ? rowCount : GroupScan.NO_COLUMN_STATS; + return rowCount != null ? rowCount : Statistic.NO_COLUMN_STATS; } @Override diff --git a/metastore/pom.xml b/metastore/pom.xml new file mode 100644 index 00000000000..f7f00060e57 --- /dev/null +++ b/metastore/pom.xml @@ -0,0 +1,75 @@ + + + + 4.0.0 + + + org.apache.drill + drill-root + 1.16.0-SNAPSHOT + + + org.apache.drill.metastore + metastore-parent + pom + metastore/Parent Pom + + + + org.apache.drill + drill-common + ${project.version} + + + org.apache.drill + drill-logical + ${project.version} + + + org.apache.drill.exec + vector + ${project.version} + + + org.apache.hadoop + hadoop-common + + + commons-codec + commons-codec + + + io.netty + netty + + + io.netty + netty-all + + + + + + + metastore-api + file-metadata + + diff --git a/pom.xml b/pom.xml index 00e7728f9ed..555817a0a53 100644 --- a/pom.xml +++ b/pom.xml @@ -3586,5 +3586,6 @@ exec drill-yarn distribution + metastore