Skip to content
Permalink
Browse files
[HUDI-3647] HoodieMetadataTableValidator: check MDT was initialized a…
…t first (#5152)

Co-authored-by: yuezhang <yuezhang@freewheel.tv>
  • Loading branch information
zhangyue19921010 and yuezhang committed Mar 30, 2022
1 parent 8b796e9 commit eae8488536c711a104c9e4f3429af3ad5c420841
Showing 1 changed file with 22 additions and 0 deletions.
@@ -357,6 +357,9 @@ public void doMetadataTableValidation() {
String basePath = metaClient.getBasePath();
Set<String> baseFilesForCleaning = Collections.emptySet();

// check metadata table is available to read.
checkMetadataTableIsAvailable();

if (cfg.skipDataFilesForCleaning) {
HoodieTimeline inflightCleaningTimeline = metaClient.getActiveTimeline().getCleanerTimeline().filterInflights();

@@ -415,6 +418,25 @@ public void doMetadataTableValidation() {
}
}

/**
* Check metadata is initialized and available to ready.
* If not we will log.warn and skip current validation.
*/
private void checkMetadataTableIsAvailable() {
try {
HoodieTableMetaClient mdtMetaClient = HoodieTableMetaClient.builder()
.setConf(jsc.hadoopConfiguration()).setBasePath(new Path(cfg.basePath, HoodieTableMetaClient.METADATA_TABLE_FOLDER_PATH).toString())
.setLoadActiveTimelineOnLoad(true)
.build();
int finishedInstants = mdtMetaClient.getActiveTimeline().filterCompletedInstants().countInstants();
if (finishedInstants == 0) {
throw new HoodieValidationException("There is no completed instant for metadata table.");
}
} catch (Exception ex) {
LOG.warn("Metadata table is not available to ready for now, ", ex);
}
}

/**
* Compare the listing partitions result between metadata table and fileSystem.
*/

0 comments on commit eae8488

Please sign in to comment.