Skip to content

Commit

Permalink
[GOBBLIN-1494]Fix flaky test on HiveSource (#3338)
Browse files Browse the repository at this point in the history
HiveSourceTest returns 2 watermarks for testGetWorkunitsAfterWatermark instead of 1.
This is due to System.getCurrentTime.millis() returning a different value when comparing against the filesystem last updated filetime. To guard against this, change the test to incorporate the workunit within the watermark by assigning it an earlier creation time, thus implying that the file was modified so it needs to be ingested again.
  • Loading branch information
Will-Lo committed Jul 23, 2021
1 parent 08db23e commit 48af4c6
Showing 1 changed file with 12 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -133,21 +133,30 @@ public void testGetWorkunitsAfterWatermark() throws Exception {

Table table1 = this.hiveMetastoreTestUtils.getLocalMetastoreClient().getTable(dbName, TEST_TABLE_1);

// Denote watermark to have a past created timestamp, so that the watermark workunit gets generated
// This is so that the test is reliable across different operating systems and not flaky to System timing differences
previousWorkUnitStates.add(ConversionHiveTestUtils.createWus(dbName, TEST_TABLE_1,
TimeUnit.MILLISECONDS.convert(table1.getCreateTime(), TimeUnit.SECONDS)));
TimeUnit.MILLISECONDS.convert(table1.getCreateTime(), TimeUnit.SECONDS)-100));

SourceState testState = new SourceState(getTestState(dbName), previousWorkUnitStates);
testState.setProp(HiveSource.HIVE_SOURCE_WATERMARKER_FACTORY_CLASS_KEY, TableLevelWatermarker.Factory.class.getName());

List<WorkUnit> workUnits = this.hiveSource.getWorkunits(testState);

Assert.assertEquals(workUnits.size(), 1);
Assert.assertEquals(workUnits.size(), 2);
WorkUnit wu = workUnits.get(0);

HiveWorkUnit hwu = new HiveWorkUnit(wu);

Assert.assertEquals(hwu.getHiveDataset().getDbAndTable().getDb(), dbName);
Assert.assertEquals(hwu.getHiveDataset().getDbAndTable().getTable(), TEST_TABLE_2);
Assert.assertEquals(hwu.getHiveDataset().getDbAndTable().getTable(), TEST_TABLE_1);

WorkUnit wu2 = workUnits.get(1);

HiveWorkUnit hwu2 = new HiveWorkUnit(wu2);

Assert.assertEquals(hwu2.getHiveDataset().getDbAndTable().getDb(), dbName);
Assert.assertEquals(hwu2.getHiveDataset().getDbAndTable().getTable(), TEST_TABLE_2);
}

@Test
Expand Down

0 comments on commit 48af4c6

Please sign in to comment.