diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java index 3207e45a11e1..b6902da8929f 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java @@ -22,7 +22,9 @@ import com.google.common.cache.LoadingCache; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; +import com.google.common.collect.SetMultimap; import com.google.common.util.concurrent.UncheckedExecutionException; import io.airlift.units.Duration; import org.weakref.jmx.Managed; @@ -41,7 +43,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.function.Function; -import java.util.stream.Collectors; import static com.facebook.presto.hive.HiveUtil.toPartitionValues; import static com.google.common.base.MoreObjects.toStringHelper; @@ -49,13 +50,15 @@ import static com.google.common.base.Throwables.throwIfInstanceOf; import static com.google.common.base.Throwables.throwIfUnchecked; import static com.google.common.cache.CacheLoader.asyncReloading; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static com.google.common.collect.ImmutableSetMultimap.toImmutableSetMultimap; import static com.google.common.collect.Iterables.transform; import static com.google.common.collect.Streams.stream; import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toMap; /** * Hive Metastore Cache @@ -69,8 +72,8 @@ public class CachingHiveMetastore private final LoadingCache> databaseNamesCache; private final LoadingCache> tableCache; private final LoadingCache>> tableNamesCache; - private final LoadingCache> tableColumnStatisticsCache; - private final LoadingCache> partitionColumnStatisticsCache; + private final LoadingCache> tableColumnStatisticsCache; + private final LoadingCache> partitionColumnStatisticsCache; private final LoadingCache>> viewNamesCache; private final LoadingCache> partitionCache; private final LoadingCache>> partitionFilterCache; @@ -124,32 +127,26 @@ private CachingHiveMetastore(ExtendedHiveMetastore delegate, ExecutorService exe .build(asyncReloading(CacheLoader.from(this::loadAllTables), executor)); tableColumnStatisticsCache = newCacheBuilder(expiresAfterWriteMillis, refreshMills, maximumSize) - .build(asyncReloading(new CacheLoader>() + .build(asyncReloading(new CacheLoader>() { @Override - public Optional load(TableColumnStatisticsCacheKey key) + public Map load(HiveTableName key) { - return loadAll(ImmutableList.of(key)).get(key); - } - - @Override - public Map> loadAll(Iterable keys) - { - return loadColumnStatistics(keys); + return loadTableColumnStatistics(key); } }, executor)); partitionColumnStatisticsCache = newCacheBuilder(expiresAfterWriteMillis, refreshMills, maximumSize) - .build(asyncReloading(new CacheLoader>() + .build(asyncReloading(new CacheLoader>() { @Override - public Optional load(PartitionColumnStatisticsCacheKey key) + public Map load(HivePartitionName key) { - return loadAll(ImmutableList.of(key)).get(key); + return loadPartitionColumnStatistics(key); } @Override - public Map> loadAll(Iterable keys) + public Map> loadAll(Iterable keys) { return loadPartitionColumnStatistics(keys); } @@ -261,100 +258,58 @@ private Optional loadTable(HiveTableName hiveTableName) } @Override - public Optional> getTableColumnStatistics(String databaseName, String tableName, Set columnNames) + public Map getTableColumnStatistics(String databaseName, String tableName) { - Map> cacheValues = - getAll(tableColumnStatisticsCache, columnNames.stream() - .map(columnName -> new TableColumnStatisticsCacheKey(databaseName, tableName, columnName)) - .collect(toList())); - - return Optional.of( - ImmutableMap.copyOf( - cacheValues.entrySet().stream() - .filter(entry -> entry.getValue().isPresent()) - .collect(toMap( - entry -> entry.getKey().getColumnName(), - entry -> entry.getValue().get())))); + return get(tableColumnStatisticsCache, new HiveTableName(databaseName, tableName)); } - private Map> loadColumnStatistics(Iterable keys) + private Map loadTableColumnStatistics(HiveTableName hiveTableName) { - if (Iterables.isEmpty(keys)) { - return ImmutableMap.of(); - } - - HiveTableName hiveTableName = stream(keys).findFirst().get().getHiveTableName(); - checkArgument(stream(keys).allMatch(key -> key.getHiveTableName().equals(hiveTableName)), "all keys must relate to same hive table"); - - Set columnNames = stream(keys).map(TableColumnStatisticsCacheKey::getColumnName).collect(Collectors.toSet()); - - Optional> columnStatistics = delegate.getTableColumnStatistics(hiveTableName.getDatabaseName(), hiveTableName.getTableName(), columnNames); - - ImmutableMap.Builder> resultMap = ImmutableMap.builder(); - for (TableColumnStatisticsCacheKey key : keys) { - if (!columnStatistics.isPresent() || !columnStatistics.get().containsKey(key.getColumnName())) { - resultMap.put(key, Optional.empty()); - } - else { - resultMap.put(key, Optional.of(columnStatistics.get().get(key.getColumnName()))); - } - } - return resultMap.build(); + return delegate.getTableColumnStatistics(hiveTableName.getDatabaseName(), hiveTableName.getTableName()); } @Override - public Optional>> getPartitionColumnStatistics(String databaseName, String tableName, Set partitionNames, Set columnNames) - { - List cacheKeys = partitionNames.stream() - .flatMap( - partitionName -> columnNames.stream().map( - columnName -> new PartitionColumnStatisticsCacheKey(databaseName, tableName, partitionName, columnName))) - .collect(toList()); - Map> cacheValues = getAll(partitionColumnStatisticsCache, cacheKeys); - - ImmutableMap.Builder> partitionsMap = ImmutableMap.builder(); - for (String partitionName : partitionNames) { - ImmutableMap.Builder columnsMap = ImmutableMap.builder(); - for (String columnName : columnNames) { - Optional cacheValue = cacheValues.get(new PartitionColumnStatisticsCacheKey(databaseName, tableName, partitionName, columnName)); - if (cacheValue.isPresent()) { - columnsMap.put(columnName, cacheValue.get()); + public Map> getPartitionColumnStatistics(String databaseName, String tableName, Set partitionNames) + { + List partitions = partitionNames.stream() + .map(partitionName -> HivePartitionName.partition(databaseName, tableName, partitionName)) + .collect(toImmutableList()); + Map> statistics = getAll(partitionColumnStatisticsCache, partitions); + return statistics.entrySet() + .stream() + .filter(entry -> !entry.getValue().isEmpty()) + .collect(toImmutableMap(entry -> entry.getKey().getPartitionName(), Entry::getValue)); + } + + private Map loadPartitionColumnStatistics(HivePartitionName partition) + { + Map> columnStatistics = delegate.getPartitionColumnStatistics( + partition.getHiveTableName().getDatabaseName(), + partition.getHiveTableName().getTableName(), + ImmutableSet.of(partition.getPartitionName())); + return columnStatistics.getOrDefault(partition.getPartitionName(), ImmutableMap.of()); + } + + private Map> loadPartitionColumnStatistics(Iterable keys) + { + SetMultimap tablePartitions = stream(keys) + .collect(toImmutableSetMultimap(HivePartitionName::getHiveTableName, key -> key)); + ImmutableMap.Builder> result = ImmutableMap.builder(); + tablePartitions.keySet().forEach(table -> { + Set partitionNames = tablePartitions.get(table).stream() + .map(HivePartitionName::getPartitionName) + .collect(toImmutableSet()); + Map> partitionStatistics = delegate.getPartitionColumnStatistics(table.getDatabaseName(), table.getTableName(), partitionNames); + for (String partitionName : partitionNames) { + if (partitionStatistics.containsKey(partitionName)) { + result.put(HivePartitionName.partition(table, partitionName), partitionStatistics.get(partitionName)); + } + else { + result.put(HivePartitionName.partition(table, partitionName), ImmutableMap.of()); } } - partitionsMap.put(partitionName, columnsMap.build()); - } - return Optional.of(partitionsMap.build()); - } - - private Map> loadPartitionColumnStatistics(Iterable keys) - { - if (Iterables.isEmpty(keys)) { - return ImmutableMap.of(); - } - PartitionColumnStatisticsCacheKey firstKey = Iterables.getFirst(keys, null); - HiveTableName hiveTableName = firstKey.getHivePartitionName().getHiveTableName(); - checkArgument(stream(keys).allMatch(key -> key.getHivePartitionName().getHiveTableName().equals(hiveTableName)), "all keys must relate to same hive table"); - Set partitionNames = stream(keys).map(key -> key.getHivePartitionName().getPartitionName()).collect(Collectors.toSet()); - Set columnNames = stream(keys).map(PartitionColumnStatisticsCacheKey::getColumnName).collect(Collectors.toSet()); - - Optional>> columnStatistics = delegate.getPartitionColumnStatistics( - hiveTableName.getDatabaseName(), - hiveTableName.getTableName(), - partitionNames, - columnNames); - - ImmutableMap.Builder> resultMap = ImmutableMap.builder(); - for (PartitionColumnStatisticsCacheKey key : keys) { - if (columnStatistics.isPresent() - && columnStatistics.get().containsKey(key.getHivePartitionName().getPartitionName()) - && columnStatistics.get().get(key.getHivePartitionName().getPartitionName()).containsKey(key.getColumnName())) { - resultMap.put(key, Optional.of(columnStatistics.get().get(key.getHivePartitionName().getPartitionName()).get(key.getColumnName()))); - } - else { - resultMap.put(key, Optional.empty()); - } - } - return resultMap.build(); + }); + return result.build(); } @Override @@ -971,93 +926,4 @@ public String toString() .toString(); } } - - private static final class TableColumnStatisticsCacheKey - { - private final HiveTableName hiveTableName; - private final String columnName; - - public TableColumnStatisticsCacheKey(String databaseName, String tableName, String columnName) - { - this.hiveTableName = HiveTableName.table( - requireNonNull(databaseName, "databaseName is null"), - requireNonNull(tableName, "tableName can not be null")); - this.columnName = requireNonNull(columnName, "columnName can not be null"); - } - - public HiveTableName getHiveTableName() - { - return hiveTableName; - } - - public String getColumnName() - { - return columnName; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - TableColumnStatisticsCacheKey that = (TableColumnStatisticsCacheKey) o; - return Objects.equals(hiveTableName, that.hiveTableName) && - Objects.equals(columnName, that.columnName); - } - - @Override - public int hashCode() - { - return Objects.hash(hiveTableName, columnName); - } - } - - private static final class PartitionColumnStatisticsCacheKey - { - private final HivePartitionName hivePartitionName; - private final String columnName; - - public PartitionColumnStatisticsCacheKey(String databaseName, String tableName, String partitionName, String columnName) - { - this.hivePartitionName = HivePartitionName.partition( - requireNonNull(databaseName, "databaseName is null"), - requireNonNull(tableName, "tableName can not be null"), - requireNonNull(partitionName, "partitionName can not be null")); - this.columnName = requireNonNull(columnName, "columnName can not be null"); - } - - public HivePartitionName getHivePartitionName() - { - return hivePartitionName; - } - - public String getColumnName() - { - return columnName; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - PartitionColumnStatisticsCacheKey that = (PartitionColumnStatisticsCacheKey) o; - return Objects.equals(hivePartitionName, that.hivePartitionName) && - Objects.equals(columnName, that.columnName); - } - - @Override - public int hashCode() - { - return Objects.hash(hivePartitionName, columnName); - } - } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/ExtendedHiveMetastore.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/ExtendedHiveMetastore.java index ad4cea93dfe0..ca0e2020234a 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/ExtendedHiveMetastore.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/ExtendedHiveMetastore.java @@ -29,9 +29,9 @@ public interface ExtendedHiveMetastore Optional
getTable(String databaseName, String tableName); - Optional> getTableColumnStatistics(String databaseName, String tableName, Set columnNames); + Map getTableColumnStatistics(String databaseName, String tableName); - Optional>> getPartitionColumnStatistics(String databaseName, String tableName, Set partitionNames, Set columnNames); + Map> getPartitionColumnStatistics(String databaseName, String tableName, Set partitionNames); Optional> getAllTables(String databaseName); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/SemiTransactionalHiveMetastore.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/SemiTransactionalHiveMetastore.java index cef9faef3fd5..52c23d1d2f2d 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/SemiTransactionalHiveMetastore.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/SemiTransactionalHiveMetastore.java @@ -150,30 +150,30 @@ public synchronized Optional
getTable(String databaseName, String tableNa } } - public synchronized Optional> getTableColumnStatistics(String databaseName, String tableName, Set columnNames) + public synchronized Map getTableColumnStatistics(String databaseName, String tableName) { checkReadable(); Action tableAction = tableActions.get(new SchemaTableName(databaseName, tableName)); if (tableAction == null) { - return delegate.getTableColumnStatistics(databaseName, tableName, columnNames); + return delegate.getTableColumnStatistics(databaseName, tableName); } switch (tableAction.getType()) { case ADD: case ALTER: case INSERT_EXISTING: case DROP: - return Optional.empty(); + return ImmutableMap.of(); default: throw new IllegalStateException("Unknown action type"); } } - public synchronized Optional>> getPartitionColumnStatistics(String databaseName, String tableName, Set partitionNames, Set columnNames) + public synchronized Map> getPartitionColumnStatistics(String databaseName, String tableName, Set partitionNames) { checkReadable(); Optional
table = getTable(databaseName, tableName); if (!table.isPresent()) { - return Optional.empty(); + return ImmutableMap.of(); } TableSource tableSource = getTableSource(databaseName, tableName); Map, Action> partitionActionsOfTable = partitionActions.computeIfAbsent(new SchemaTableName(databaseName, tableName), k -> new HashMap<>()); @@ -199,14 +199,14 @@ public synchronized Optional>> get } } - Optional>> delegateResult = delegate.getPartitionColumnStatistics(databaseName, tableName, partitionNamesToQuery.build(), columnNames); - if (delegateResult.isPresent()) { - resultBuilder.putAll(delegateResult.get()); + Map> delegateResult = delegate.getPartitionColumnStatistics(databaseName, tableName, partitionNamesToQuery.build()); + if (!delegateResult.isEmpty()) { + resultBuilder.putAll(delegateResult); } else { partitionNamesToQuery.build().forEach(partionName -> resultBuilder.put(partionName, ImmutableMap.of())); } - return Optional.of(resultBuilder.build()); + return resultBuilder.build(); } /** diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastore.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastore.java index c8e83ed9c9be..2c2c1ef48e8c 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastore.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastore.java @@ -270,15 +270,15 @@ public synchronized Optional
getTable(String databaseName, String tableNa } @Override - public Optional> getTableColumnStatistics(String databaseName, String tableName, Set columnNames) + public Map getTableColumnStatistics(String databaseName, String tableName) { - return Optional.of(ImmutableMap.of()); + return ImmutableMap.of(); } @Override - public Optional>> getPartitionColumnStatistics(String databaseName, String tableName, Set partitionNames, Set columnNames) + public Map> getPartitionColumnStatistics(String databaseName, String tableName, Set partitionNames) { - return Optional.of(ImmutableMap.of()); + return ImmutableMap.of(); } private Table getRequiredTable(String databaseName, String tableName) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java index 6233dc231efb..6a401029e62f 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java @@ -217,15 +217,15 @@ private Table getTableOrElseThrow(String databaseName, String tableName) } @Override - public Optional> getTableColumnStatistics(String databaseName, String tableName, Set columnNames) + public Map getTableColumnStatistics(String databaseName, String tableName) { - return Optional.of(ImmutableMap.of()); + return ImmutableMap.of(); } @Override - public Optional>> getPartitionColumnStatistics(String databaseName, String tableName, Set partitionNames, Set columnNames) + public Map> getPartitionColumnStatistics(String databaseName, String tableName, Set partitionNames) { - return Optional.of(ImmutableMap.of()); + return ImmutableMap.of(); } @Override diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/BridgingHiveMetastore.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/BridgingHiveMetastore.java index 26b22c431a38..377843843ac9 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/BridgingHiveMetastore.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/BridgingHiveMetastore.java @@ -16,6 +16,7 @@ import com.facebook.presto.hive.HiveType; import com.facebook.presto.hive.HiveUtil; import com.facebook.presto.hive.PartitionNotFoundException; +import com.facebook.presto.hive.metastore.Column; import com.facebook.presto.hive.metastore.Database; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.HiveColumnStatistics; @@ -47,6 +48,8 @@ import static com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil.toMetastoreApiPrivilegeGrantInfo; import static com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil.toMetastoreApiTable; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.collect.ImmutableSet.toImmutableSet; import static java.util.Objects.requireNonNull; import static java.util.function.UnaryOperator.identity; @@ -80,29 +83,35 @@ public Optional
getTable(String databaseName, String tableName) } @Override - public Optional> getTableColumnStatistics(String databaseName, String tableName, Set columnNames) + public Map getTableColumnStatistics(String databaseName, String tableName) { - return delegate.getTableColumnStatistics(databaseName, tableName, columnNames).map(this::groupStatisticsByColumn); + Table table = getTable(databaseName, tableName) + .orElseThrow(() -> new TableNotFoundException(new SchemaTableName(databaseName, tableName))); + Set dataColumns = table.getDataColumns().stream() + .map(Column::getName) + .collect(toImmutableSet()); + return groupStatisticsByColumn(delegate.getTableColumnStatistics(databaseName, tableName, dataColumns)); } @Override - public Optional>> getPartitionColumnStatistics(String databaseName, String tableName, Set partitionNames, Set columnNames) + public Map> getPartitionColumnStatistics(String databaseName, String tableName, Set partitionNames) { - return delegate.getPartitionColumnStatistics(databaseName, tableName, partitionNames, columnNames).map( - statistics -> ImmutableMap.copyOf( - statistics.entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - entry -> groupStatisticsByColumn(entry.getValue()))))); + Table table = getTable(databaseName, tableName) + .orElseThrow(() -> new TableNotFoundException(new SchemaTableName(databaseName, tableName))); + Set dataColumns = table.getDataColumns().stream() + .map(Column::getName) + .collect(toImmutableSet()); + Map> statistics = delegate.getPartitionColumnStatistics(databaseName, tableName, partitionNames, dataColumns); + return statistics.entrySet() + .stream() + .filter(entry -> !entry.getValue().isEmpty()) + .collect(toImmutableMap(Map.Entry::getKey, entry -> groupStatisticsByColumn(entry.getValue()))); } private Map groupStatisticsByColumn(Set statistics) { - return ImmutableMap.copyOf( - statistics.stream() - .collect(Collectors.toMap( - ColumnStatisticsObj::getColName, - ThriftMetastoreUtil::fromMetastoreApiColumnStatistics))); + return statistics.stream() + .collect(toImmutableMap(ColumnStatisticsObj::getColName, ThriftMetastoreUtil::fromMetastoreApiColumnStatistics)); } @Override diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastore.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastore.java index 007b2c2dc801..631091d7e646 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastore.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastore.java @@ -72,9 +72,9 @@ public interface HiveMetastore Optional
getTable(String databaseName, String tableName); - Optional> getTableColumnStatistics(String databaseName, String tableName, Set columnNames); + Set getTableColumnStatistics(String databaseName, String tableName, Set columnNames); - Optional>> getPartitionColumnStatistics(String databaseName, String tableName, Set partitionNames, Set columnNames); + Map> getPartitionColumnStatistics(String databaseName, String tableName, Set partitionNames, Set columnNames); Set getRoles(String user); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastore.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastore.java index 0aaa90992ee7..8a9583d1a474 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastore.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastore.java @@ -25,6 +25,7 @@ import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.TableNotFoundException; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import org.apache.hadoop.hive.metastore.TableType; @@ -54,6 +55,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.concurrent.Callable; @@ -70,12 +72,12 @@ import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Throwables.throwIfUnchecked; +import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.Iterables.getOnlyElement; import static com.google.common.collect.Sets.newHashSet; import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static java.util.function.Function.identity; -import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toSet; import static org.apache.hadoop.hive.metastore.api.HiveObjectType.DATABASE; import static org.apache.hadoop.hive.metastore.api.HiveObjectType.TABLE; @@ -229,7 +231,7 @@ private static boolean isPrestoView(Table table) } @Override - public Optional> getTableColumnStatistics(String databaseName, String tableName, Set columnNames) + public Set getTableColumnStatistics(String databaseName, String tableName, Set columnNames) { try { return retry() @@ -237,12 +239,12 @@ public Optional> getTableColumnStatistics(String databa .stopOnIllegalExceptions() .run("getTableColumnStatistics", stats.getGetTableColumnStatistics().wrap(() -> { try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) { - return Optional.of(ImmutableSet.copyOf(client.getTableColumnStatistics(databaseName, tableName, ImmutableList.copyOf(columnNames)))); + return ImmutableSet.copyOf(client.getTableColumnStatistics(databaseName, tableName, ImmutableList.copyOf(columnNames))); } })); } catch (NoSuchObjectException e) { - return Optional.empty(); + return ImmutableSet.of(); } catch (TException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); @@ -253,7 +255,7 @@ public Optional> getTableColumnStatistics(String databa } @Override - public Optional>> getPartitionColumnStatistics(String databaseName, String tableName, Set partitionNames, Set columnNames) + public Map> getPartitionColumnStatistics(String databaseName, String tableName, Set partitionNames, Set columnNames) { try { return retry() @@ -262,16 +264,14 @@ public Optional>> getPartitionColumnStatist .run("getPartitionColumnStatistics", stats.getGetPartitionColumnStatistics().wrap(() -> { try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) { Map> partitionColumnStatistics = client.getPartitionColumnStatistics(databaseName, tableName, ImmutableList.copyOf(partitionNames), ImmutableList.copyOf(columnNames)); - return Optional.of(partitionColumnStatistics.entrySet() + return partitionColumnStatistics.entrySet() .stream() - .collect(toMap( - Map.Entry::getKey, - entry -> ImmutableSet.copyOf(entry.getValue())))); + .collect(toImmutableMap(Entry::getKey, entry -> ImmutableSet.copyOf(entry.getValue()))); } })); } catch (NoSuchObjectException e) { - return Optional.empty(); + return ImmutableMap.of(); } catch (TException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/statistics/MetastoreHiveStatisticsProvider.java b/presto-hive/src/main/java/com/facebook/presto/hive/statistics/MetastoreHiveStatisticsProvider.java index 203e396f168a..3c8abb9175b2 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/statistics/MetastoreHiveStatisticsProvider.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/statistics/MetastoreHiveStatisticsProvider.java @@ -39,6 +39,7 @@ import com.facebook.presto.spi.type.TypeManager; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import org.joda.time.DateTimeZone; import java.math.BigDecimal; @@ -46,16 +47,13 @@ import java.time.LocalDate; import java.util.Collection; import java.util.Comparator; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.OptionalDouble; import java.util.OptionalLong; import java.util.PrimitiveIterator; -import java.util.Set; import java.util.function.Function; -import java.util.stream.Collectors; import java.util.stream.DoubleStream; import static com.facebook.presto.hive.HiveSessionProperties.isStatisticsEnabled; @@ -70,11 +68,9 @@ import static com.facebook.presto.spi.type.TinyintType.TINYINT; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; -import static com.google.common.collect.ImmutableSet.toImmutableSet; import static java.lang.Float.floatToRawIntBits; import static java.lang.String.format; import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.toList; public class MetastoreHiveStatisticsProvider implements HiveStatisticsProvider @@ -240,7 +236,7 @@ private Estimate calculateRowsCount(Map partitionSt .map(stats -> stats.getBasicStatistics().getRowCount()) .filter(OptionalLong::isPresent) .map(OptionalLong::getAsLong) - .collect(toList()); + .collect(toImmutableList()); long knownPartitionRowCountsSum = knownPartitionRowCounts.stream().mapToLong(a -> a).sum(); long partitionsWithStatsCount = knownPartitionRowCounts.size(); @@ -382,32 +378,23 @@ private Map getPartitionsStatistics(HiveTableHandle } if (unpartitioned) { - return ImmutableMap.of(HivePartition.UNPARTITIONED_ID, getTableStatistics(tableHandle.getSchemaTableName(), tableColumns.keySet())); + return ImmutableMap.of(HivePartition.UNPARTITIONED_ID, getTableStatistics(tableHandle.getSchemaTableName())); } else { - return getPartitionsStatistics(tableHandle.getSchemaTableName(), hivePartitions, listNonPartitioningColumns(tableColumns)); + return getPartitionsStatistics(tableHandle.getSchemaTableName(), hivePartitions); } } - private static Set listNonPartitioningColumns(Map tableColumns) - { - return tableColumns.entrySet().stream() - .filter(entry -> !((HiveColumnHandle) entry.getValue()).isPartitionKey()) - .map(Map.Entry::getKey) - .collect(toImmutableSet()); - } - - private Map getPartitionsStatistics(SchemaTableName schemaTableName, List hivePartitions, Set tableColumns) + private Map getPartitionsStatistics(SchemaTableName schemaTableName, List hivePartitions) { String databaseName = schemaTableName.getSchemaName(); String tableName = schemaTableName.getTableName(); ImmutableMap.Builder resultMap = ImmutableMap.builder(); - List partitionNames = hivePartitions.stream().map(HivePartition::getPartitionId).collect(Collectors.toList()); + List partitionNames = hivePartitions.stream().map(HivePartition::getPartitionId).collect(toImmutableList()); Map> partitionColumnStatisticsMap = - metastore.getPartitionColumnStatistics(databaseName, tableName, new HashSet<>(partitionNames), tableColumns) - .orElse(ImmutableMap.of()); + metastore.getPartitionColumnStatistics(databaseName, tableName, ImmutableSet.copyOf(partitionNames)); Map> partitionsByNames = metastore.getPartitionsByNames(databaseName, tableName, partitionNames); for (String partitionName : partitionNames) { @@ -421,14 +408,14 @@ private Map getPartitionsStatistics(SchemaTableName return resultMap.build(); } - private PartitionStatistics getTableStatistics(SchemaTableName schemaTableName, Set tableColumns) + private PartitionStatistics getTableStatistics(SchemaTableName schemaTableName) { String databaseName = schemaTableName.getSchemaName(); String tableName = schemaTableName.getTableName(); Table table = metastore.getTable(databaseName, tableName) .orElseThrow(() -> new IllegalArgumentException(format("Could not get metadata for table %s.%s", databaseName, tableName))); - Map tableColumnStatistics = metastore.getTableColumnStatistics(databaseName, tableName, tableColumns).orElse(ImmutableMap.of()); + Map tableColumnStatistics = metastore.getTableColumnStatistics(databaseName, tableName); return readStatisticsFromParameters(table.getParameters(), tableColumnStatistics); } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/InMemoryHiveMetastore.java b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/InMemoryHiveMetastore.java index 1e543223e9f3..4c20e86aaadd 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/InMemoryHiveMetastore.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/InMemoryHiveMetastore.java @@ -441,18 +441,18 @@ public synchronized Optional
getTable(String databaseName, String tableNa } @Override - public synchronized Optional> getTableColumnStatistics(String databaseName, String tableName, Set columnNames) + public synchronized Set getTableColumnStatistics(String databaseName, String tableName, Set columnNames) { SchemaTableName schemaTableName = new SchemaTableName(databaseName, tableName); if (!columnStatistics.containsKey(schemaTableName)) { - return Optional.empty(); + return ImmutableSet.of(); } Map columnStatisticsMap = columnStatistics.get(schemaTableName); - return Optional.of(columnNames.stream() + return columnNames.stream() .filter(columnStatisticsMap::containsKey) .map(columnStatisticsMap::get) - .collect(toImmutableSet())); + .collect(toImmutableSet()); } public synchronized void setColumnStatistics(String databaseName, String tableName, String columnName, ColumnStatisticsObj columnStatisticsObj) @@ -463,7 +463,7 @@ public synchronized void setColumnStatistics(String databaseName, String tableNa } @Override - public synchronized Optional>> getPartitionColumnStatistics(String databaseName, String tableName, Set partitionNames, Set columnNames) + public synchronized Map> getPartitionColumnStatistics(String databaseName, String tableName, Set partitionNames, Set columnNames) { ImmutableMap.Builder> result = ImmutableMap.builder(); for (String partitionName : partitionNames) { @@ -480,7 +480,7 @@ public synchronized Optional>> getPartition .map(columnStatistics::get) .collect(toImmutableSet())); } - return Optional.of(result.build()); + return result.build(); } public synchronized void setPartitionColumnStatistics(String databaseName, String tableName, String partitionName, String columnName, ColumnStatisticsObj columnStatisticsObj)