Skip to content
Permalink
Browse files
[HUDI-3183] Wrong result of HoodieArchivedTimeline loadInstants with …
…TimeRangeFilter (#4521)


Co-authored-by: yuezhang <yuezhang@freewheel.tv>
  • Loading branch information
zhangyue19921010 and yuezhang committed Jan 7, 2022
1 parent d7afc58 commit b2b23f5d3a14fae721df34a162e0ed717efe488f
Showing 2 changed files with 48 additions and 1 deletion.
@@ -251,6 +251,53 @@ public void testShowArchivedCommits() throws Exception {
assertEquals(expected, got);
}

@Test
public void testShowArchivedCommitsWithMultiCommitsFile() throws Exception {
// Generate archive
HoodieWriteConfig cfg = HoodieWriteConfig.newBuilder().withPath(tablePath1)
.withSchema(HoodieTestCommitMetadataGenerator.TRIP_EXAMPLE_SCHEMA).withParallelism(2, 2)
.withCompactionConfig(HoodieCompactionConfig.newBuilder().retainCommits(1).archiveCommitsWith(2, 3).build())
.withFileSystemViewConfig(FileSystemViewStorageConfig.newBuilder()
.withRemoteServerPort(timelineServicePort).build())
.withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(false).build())
.forTable("test-trip-table").build();

// generate data and metadata
Map<String, Integer[]> data = new LinkedHashMap<>();

for (int i = 194; i >= 154; i--) {
data.put(String.valueOf(i), new Integer[] {i, i});
}

for (Map.Entry<String, Integer[]> entry : data.entrySet()) {
String key = entry.getKey();
Integer[] value = entry.getValue();
HoodieTestCommitMetadataGenerator.createCommitFileWithMetadata(tablePath1, key, hadoopConf(),
Option.of(value[0]), Option.of(value[1]));
// archive
metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());
HoodieSparkTable table = HoodieSparkTable.create(cfg, context(), metaClient);

// need to create multi archive files
HoodieTimelineArchiveLog archiveLog = new HoodieTimelineArchiveLog(cfg, table);
archiveLog.archiveIfRequired(context());
}

CommandResult cr = shell().executeCommand(String.format("commits showarchived --startTs %s --endTs %s", "160", "174"));
assertTrue(cr.isSuccess());
assertEquals(3, metaClient.reloadActiveTimeline().getCommitsTimeline().countInstants(),
"There should 3 instants not be archived!");

Map<String, Integer[]> data2 = new LinkedHashMap<>();
for (int i = 174; i >= 161; i--) {
data2.put(String.valueOf(i), new Integer[] {i, i});
}
String expected = generateExpectData(1, data2);
expected = removeNonWordAndStripSpace(expected);
String got = removeNonWordAndStripSpace(cr.getResult().toString());
assertEquals(expected, got);
}

/**
* Test case of 'commit showpartitions' command.
*/
@@ -67,7 +67,7 @@
*/
public class HoodieArchivedTimeline extends HoodieDefaultTimeline {
private static final Pattern ARCHIVE_FILE_PATTERN =
Pattern.compile("^\\.commits_\\.archive\\.([0-9]*)$");
Pattern.compile("^\\.commits_\\.archive\\.([0-9]+).*");

private static final String HOODIE_COMMIT_ARCHIVE_LOG_FILE_PREFIX = "commits";
private static final String ACTION_TYPE_KEY = "actionType";

0 comments on commit b2b23f5

Please sign in to comment.