Skip to content

Commit

Permalink
Rename PartitionVersionFetcher to PartitionMutator
Browse files Browse the repository at this point in the history
  • Loading branch information
highker committed Oct 7, 2020
1 parent 55508df commit 9870fb8
Show file tree
Hide file tree
Showing 13 changed files with 45 additions and 43 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,14 @@
*/
package com.facebook.presto.hive;

import com.facebook.presto.hive.metastore.Partition.Builder;
import org.apache.hadoop.hive.metastore.api.Partition;

import java.util.Optional;

public interface PartitionVersionFetcher
public interface PartitionMutator
{
Optional<Integer> getPartitionVersion(Partition partition);
/**
* Mutate the provided {@param builder} based on the given {@param partition}
*
*/
void mutate(Builder builder, Partition partition);
}
Original file line number Diff line number Diff line change
Expand Up @@ -13,22 +13,21 @@
*/
package com.facebook.presto.hive.metastore;

import com.facebook.presto.hive.PartitionVersionFetcher;
import com.facebook.presto.hive.PartitionMutator;
import com.facebook.presto.hive.metastore.Partition.Builder;
import org.apache.hadoop.hive.metastore.api.Partition;

import javax.inject.Inject;

import java.util.Optional;

public class HivePartitionVersionFetcher
implements PartitionVersionFetcher
public class HivePartitionMutator
implements PartitionMutator
{
@Inject
public HivePartitionVersionFetcher() {}
public HivePartitionMutator() {}

@Override
public Optional<Integer> getPartitionVersion(Partition partition)
public void mutate(Builder builder, Partition partition)
{
return Optional.empty();
// no-op
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
import com.facebook.presto.common.predicate.Domain;
import com.facebook.presto.common.type.Type;
import com.facebook.presto.hive.HiveType;
import com.facebook.presto.hive.PartitionVersionFetcher;
import com.facebook.presto.hive.PartitionMutator;
import com.facebook.presto.hive.metastore.Column;
import com.facebook.presto.hive.metastore.Database;
import com.facebook.presto.hive.metastore.ExtendedHiveMetastore;
Expand Down Expand Up @@ -64,13 +64,13 @@ public class BridgingHiveMetastore
implements ExtendedHiveMetastore
{
private final HiveMetastore delegate;
private final PartitionVersionFetcher partitionVersionFetcher;
private final PartitionMutator partitionMutator;

@Inject
public BridgingHiveMetastore(HiveMetastore delegate, PartitionVersionFetcher partitionVersionFetcher)
public BridgingHiveMetastore(HiveMetastore delegate, PartitionMutator partitionMutator)
{
this.delegate = delegate;
this.partitionVersionFetcher = partitionVersionFetcher;
this.partitionMutator = partitionMutator;
}

@Override
Expand Down Expand Up @@ -250,7 +250,7 @@ private void alterTable(String databaseName, String tableName, org.apache.hadoop
@Override
public Optional<Partition> getPartition(String databaseName, String tableName, List<String> partitionValues)
{
return delegate.getPartition(databaseName, tableName, partitionValues).map(partition -> fromMetastoreApiPartition(partition, partitionVersionFetcher));
return delegate.getPartition(databaseName, tableName, partitionValues).map(partition -> fromMetastoreApiPartition(partition, partitionMutator));
}

@Override
Expand Down Expand Up @@ -287,7 +287,7 @@ public Map<String, Optional<Partition>> getPartitionsByNames(String databaseName
Map<String, List<String>> partitionNameToPartitionValuesMap = partitionNames.stream()
.collect(Collectors.toMap(identity(), MetastoreUtil::toPartitionValues));
Map<List<String>, Partition> partitionValuesToPartitionMap = delegate.getPartitionsByNames(databaseName, tableName, partitionNames).stream()
.map(partition -> fromMetastoreApiPartition(partition, partitionVersionFetcher))
.map(partition -> fromMetastoreApiPartition(partition, partitionMutator))
.collect(Collectors.toMap(Partition::getValues, identity()));
ImmutableMap.Builder<String, Optional<Partition>> resultBuilder = ImmutableMap.builder();
for (Map.Entry<String, List<String>> entry : partitionNameToPartitionValuesMap.entrySet()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

import com.facebook.presto.hive.HiveBucketProperty;
import com.facebook.presto.hive.HiveType;
import com.facebook.presto.hive.PartitionVersionFetcher;
import com.facebook.presto.hive.PartitionMutator;
import com.facebook.presto.hive.metastore.Column;
import com.facebook.presto.hive.metastore.Database;
import com.facebook.presto.hive.metastore.HiveColumnStatistics;
Expand Down Expand Up @@ -450,7 +450,7 @@ private static SerDeInfo getSerdeInfo(org.apache.hadoop.hive.metastore.api.Table

return serdeInfo;
}
public static Partition fromMetastoreApiPartition(org.apache.hadoop.hive.metastore.api.Partition partition, PartitionVersionFetcher partitionVersionFetcher)
public static Partition fromMetastoreApiPartition(org.apache.hadoop.hive.metastore.api.Partition partition, PartitionMutator partitionMutator)
{
StorageDescriptor storageDescriptor = partition.getSd();
if (storageDescriptor == null) {
Expand All @@ -466,8 +466,8 @@ public static Partition fromMetastoreApiPartition(org.apache.hadoop.hive.metasto
.collect(toList()))
.setParameters(partition.getParameters());

// set the partition version if available
partitionVersionFetcher.getPartitionVersion(partition).ifPresent(partitionBuilder::setPartitionVersion);
// mutate apache partition to Presto partition
partitionMutator.mutate(partitionBuilder, partition);

fromMetastoreApiStorageDescriptor(storageDescriptor, partitionBuilder.getStorageBuilder(), format("%s.%s", partition.getTableName(), partition.getValues()));

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
package com.facebook.presto.hive.metastore;

import com.facebook.presto.hive.MockHiveMetastore;
import com.facebook.presto.hive.PartitionVersionFetcher;
import com.facebook.presto.hive.PartitionMutator;
import com.facebook.presto.hive.metastore.CachingHiveMetastore.MetastoreCacheScope;
import com.facebook.presto.hive.metastore.thrift.BridgingHiveMetastore;
import com.facebook.presto.hive.metastore.thrift.HiveCluster;
Expand Down Expand Up @@ -62,9 +62,9 @@ public void setUp()
MockHiveCluster mockHiveCluster = new MockHiveCluster(mockClient);
ListeningExecutorService executor = listeningDecorator(newCachedThreadPool(daemonThreadsNamed("test-%s")));
ThriftHiveMetastore thriftHiveMetastore = new ThriftHiveMetastore(mockHiveCluster);
PartitionVersionFetcher hivePartitionVersionFetcher = new HivePartitionVersionFetcher();
PartitionMutator hivePartitionMutator = new HivePartitionMutator();
metastore = new CachingHiveMetastore(
new BridgingHiveMetastore(thriftHiveMetastore, hivePartitionVersionFetcher),
new BridgingHiveMetastore(thriftHiveMetastore, hivePartitionMutator),
executor,
new Duration(5, TimeUnit.MINUTES),
new Duration(1, TimeUnit.MINUTES),
Expand Down Expand Up @@ -178,10 +178,10 @@ public void testCachingWithPartitionVersioning()
MockHiveCluster mockHiveCluster = new MockHiveCluster(mockClient);
ListeningExecutorService executor = listeningDecorator(newCachedThreadPool(daemonThreadsNamed("partition-versioning-test-%s")));
MockHiveMetastore mockHiveMetastore = new MockHiveMetastore(mockHiveCluster);
PartitionVersionFetcher hivePartitionVersionFetcher = new HivePartitionVersionFetcher();
PartitionMutator hivePartitionMutator = new HivePartitionMutator();
boolean partitionVersioningEnabled = true;
CachingHiveMetastore partitionCachingEnabledmetastore = new CachingHiveMetastore(
new BridgingHiveMetastore(mockHiveMetastore, hivePartitionVersionFetcher),
new BridgingHiveMetastore(mockHiveMetastore, hivePartitionMutator),
executor,
new Duration(5, TimeUnit.MINUTES),
new Duration(1, TimeUnit.MINUTES),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
*/
package com.facebook.presto.hive.metastore;

import com.facebook.presto.hive.PartitionVersionFetcher;
import com.facebook.presto.hive.PartitionMutator;
import com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
Expand Down Expand Up @@ -119,7 +119,7 @@ public class TestHiveMetastoreUtil
1234567893,
TEST_STORAGE_DESCRIPTOR_WITH_UNSUPPORTED_FIELDS,
ImmutableMap.of("k1", "v1", "k2", "v2", "k3", "v3"));
private static final PartitionVersionFetcher TEST_PARTITION_VERSION_FETCHER = new HivePartitionVersionFetcher();
private static final PartitionMutator TEST_PARTITION_VERSION_FETCHER = new HivePartitionMutator();

static {
TEST_STORAGE_DESCRIPTOR_WITH_UNSUPPORTED_FIELDS.setSkewedInfo(new SkewedInfo(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import com.facebook.presto.hive.cache.HiveCachingHdfsConfiguration;
import com.facebook.presto.hive.datasink.DataSinkFactory;
import com.facebook.presto.hive.datasink.OutputStreamDataSinkFactory;
import com.facebook.presto.hive.metastore.HivePartitionVersionFetcher;
import com.facebook.presto.hive.metastore.HivePartitionMutator;
import com.facebook.presto.hive.orc.DwrfBatchPageSourceFactory;
import com.facebook.presto.hive.orc.DwrfSelectivePageSourceFactory;
import com.facebook.presto.hive.orc.OrcBatchPageSourceFactory;
Expand Down Expand Up @@ -195,7 +195,7 @@ public void configure(Binder binder)
binder.bind(HiveEncryptionInformationProvider.class).in(Scopes.SINGLETON);
newSetBinder(binder, EncryptionInformationSource.class);

binder.bind(PartitionVersionFetcher.class).to(HivePartitionVersionFetcher.class).in(Scopes.SINGLETON);
binder.bind(PartitionMutator.class).to(HivePartitionMutator.class).in(Scopes.SINGLETON);
}

@ForHiveClient
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
import com.facebook.presto.hive.metastore.Column;
import com.facebook.presto.hive.metastore.ExtendedHiveMetastore;
import com.facebook.presto.hive.metastore.HiveColumnStatistics;
import com.facebook.presto.hive.metastore.HivePartitionVersionFetcher;
import com.facebook.presto.hive.metastore.HivePartitionMutator;
import com.facebook.presto.hive.metastore.HivePrivilegeInfo;
import com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege;
import com.facebook.presto.hive.metastore.Partition;
Expand Down Expand Up @@ -913,7 +913,7 @@ protected final void setup(String host, int port, String databaseName, String ti

HiveCluster hiveCluster = new TestingHiveCluster(metastoreClientConfig, host, port);
ExtendedHiveMetastore metastore = new CachingHiveMetastore(
new BridgingHiveMetastore(new ThriftHiveMetastore(hiveCluster), new HivePartitionVersionFetcher()),
new BridgingHiveMetastore(new ThriftHiveMetastore(hiveCluster), new HivePartitionMutator()),
executor,
Duration.valueOf("1m"),
Duration.valueOf("15s"),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
import com.facebook.presto.hive.metastore.CachingHiveMetastore;
import com.facebook.presto.hive.metastore.Database;
import com.facebook.presto.hive.metastore.ExtendedHiveMetastore;
import com.facebook.presto.hive.metastore.HivePartitionVersionFetcher;
import com.facebook.presto.hive.metastore.HivePartitionMutator;
import com.facebook.presto.hive.metastore.PrincipalPrivileges;
import com.facebook.presto.hive.metastore.Table;
import com.facebook.presto.hive.metastore.thrift.BridgingHiveMetastore;
Expand Down Expand Up @@ -182,7 +182,7 @@ protected void setup(String host, int port, String databaseName, BiFunction<Hive

hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, metastoreClientConfig, new NoHdfsAuthentication());
metastoreClient = new TestingHiveMetastore(
new BridgingHiveMetastore(new ThriftHiveMetastore(hiveCluster), new HivePartitionVersionFetcher()),
new BridgingHiveMetastore(new ThriftHiveMetastore(hiveCluster), new HivePartitionMutator()),
executor,
metastoreClientConfig,
getBasePath(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
package com.facebook.presto.hive;

import com.facebook.presto.hive.metastore.ExtendedHiveMetastore;
import com.facebook.presto.hive.metastore.HivePartitionVersionFetcher;
import com.facebook.presto.hive.metastore.HivePartitionMutator;
import com.facebook.presto.hive.metastore.thrift.BridgingHiveMetastore;
import com.facebook.presto.hive.metastore.thrift.InMemoryHiveMetastore;

Expand All @@ -28,7 +28,7 @@ protected ExtendedHiveMetastore createMetastore(File tempDir)
{
File baseDir = new File(tempDir, "metastore");
InMemoryHiveMetastore hiveMetastore = new InMemoryHiveMetastore(baseDir);
return new BridgingHiveMetastore(hiveMetastore, new HivePartitionVersionFetcher());
return new BridgingHiveMetastore(hiveMetastore, new HivePartitionMutator());
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
package com.facebook.presto.hive;

import com.facebook.presto.hive.metastore.ExtendedHiveMetastore;
import com.facebook.presto.hive.metastore.HivePartitionVersionFetcher;
import com.facebook.presto.hive.metastore.HivePartitionMutator;
import com.facebook.presto.hive.metastore.thrift.BridgingHiveMetastore;
import com.facebook.presto.hive.metastore.thrift.InMemoryHiveMetastore;
import com.google.common.collect.ImmutableSet;
Expand Down Expand Up @@ -43,7 +43,7 @@ protected ExtendedHiveMetastore createMetastore(File tempDir)
{
File baseDir = new File(tempDir, "metastore");
InMemoryHiveMetastore hiveMetastore = new InMemoryHiveMetastore(baseDir);
return new BridgingHiveMetastore(hiveMetastore, new HivePartitionVersionFetcher());
return new BridgingHiveMetastore(hiveMetastore, new HivePartitionMutator());
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
import com.facebook.presto.hive.datasink.OutputStreamDataSinkFactory;
import com.facebook.presto.hive.metastore.Database;
import com.facebook.presto.hive.metastore.ExtendedHiveMetastore;
import com.facebook.presto.hive.metastore.HivePartitionVersionFetcher;
import com.facebook.presto.hive.metastore.HivePartitionMutator;
import com.facebook.presto.hive.metastore.Partition;
import com.facebook.presto.hive.metastore.thrift.BridgingHiveMetastore;
import com.facebook.presto.hive.metastore.thrift.InMemoryHiveMetastore;
Expand Down Expand Up @@ -98,7 +98,7 @@ public class TestHiveMetadataFileFormatEncryptionSettings
public void setup()
{
baseDirectory = new File(Files.createTempDir(), "metastore");
metastore = new BridgingHiveMetastore(new InMemoryHiveMetastore(baseDirectory), new HivePartitionVersionFetcher());
metastore = new BridgingHiveMetastore(new InMemoryHiveMetastore(baseDirectory), new HivePartitionMutator());
executor = newCachedThreadPool(daemonThreadsNamed("hive-encryption-test-%s"));
transactionManager = new HiveTransactionManager();
metadataFactory = new HiveMetadataFactory(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
import com.facebook.presto.hive.metastore.Database;
import com.facebook.presto.hive.metastore.ExtendedHiveMetastore;
import com.facebook.presto.hive.metastore.HivePageSinkMetadata;
import com.facebook.presto.hive.metastore.HivePartitionVersionFetcher;
import com.facebook.presto.hive.metastore.HivePartitionMutator;
import com.facebook.presto.hive.metastore.HivePrivilegeInfo;
import com.facebook.presto.hive.metastore.HiveTableName;
import com.facebook.presto.hive.metastore.Partition;
Expand Down Expand Up @@ -75,7 +75,7 @@ public static TestingSemiTransactionalHiveMetastore create()
HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(config, metastoreClientConfig), ImmutableSet.of());
HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, metastoreClientConfig, new NoHdfsAuthentication());
HiveCluster hiveCluster = new TestingHiveCluster(metastoreClientConfig, HOST, PORT);
ExtendedHiveMetastore delegate = new BridgingHiveMetastore(new ThriftHiveMetastore(hiveCluster), new HivePartitionVersionFetcher());
ExtendedHiveMetastore delegate = new BridgingHiveMetastore(new ThriftHiveMetastore(hiveCluster), new HivePartitionMutator());
ExecutorService executor = newCachedThreadPool(daemonThreadsNamed("hive-%s"));
ListeningExecutorService renameExecutor = listeningDecorator(executor);

Expand Down

0 comments on commit 9870fb8

Please sign in to comment.