Skip to content
Permalink
Browse files
[HUDI-3876] Fixing fetching partitions in GlueSyncClient (#5318)
  • Loading branch information
nsivabalan committed Apr 14, 2022
1 parent 571cbe4 commit a081c2b9b54dcf5720230a2c7e0229c765208e24
Showing 1 changed file with 13 additions and 7 deletions.
@@ -94,13 +94,19 @@ public AWSGlueCatalogSyncClient(HiveSyncConfig syncConfig, Configuration hadoopC
@Override
public List<Partition> getAllPartitions(String tableName) {
try {
GetPartitionsRequest request = new GetPartitionsRequest();
request.withDatabaseName(databaseName).withTableName(tableName);
GetPartitionsResult result = awsGlue.getPartitions(request);
return result.getPartitions()
.stream()
.map(p -> new Partition(p.getValues(), p.getStorageDescriptor().getLocation()))
.collect(Collectors.toList());
List<Partition> partitions = new ArrayList<>();
String nextToken = null;
do {
GetPartitionsResult result = awsGlue.getPartitions(new GetPartitionsRequest()
.withDatabaseName(databaseName)
.withTableName(tableName)
.withNextToken(nextToken));
partitions.addAll(result.getPartitions().stream()
.map(p -> new Partition(p.getValues(), p.getStorageDescriptor().getLocation()))
.collect(Collectors.toList()));
nextToken = result.getNextToken();
} while (nextToken != null);
return partitions;
} catch (Exception e) {
throw new HoodieGlueSyncException("Failed to get all partitions for table " + tableId(databaseName, tableName), e);
}

0 comments on commit a081c2b

Please sign in to comment.