From 48af4c61c4780740b63931901c756e5a184e763b Mon Sep 17 00:00:00 2001 From: William Lo Date: Thu, 22 Jul 2021 17:57:22 -0700 Subject: [PATCH] [GOBBLIN-1494]Fix flaky test on HiveSource (#3338) HiveSourceTest returns 2 watermarks for testGetWorkunitsAfterWatermark instead of 1. This is due to System.getCurrentTime.millis() returning a different value when comparing against the filesystem last updated filetime. To guard against this, change the test to incorporate the workunit within the watermark by assigning it an earlier creation time, thus implying that the file was modified so it needs to be ingested again. --- .../conversion/hive/HiveSourceTest.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/conversion/hive/HiveSourceTest.java b/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/conversion/hive/HiveSourceTest.java index 4a1da03889e..2351fd35c66 100644 --- a/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/conversion/hive/HiveSourceTest.java +++ b/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/conversion/hive/HiveSourceTest.java @@ -133,21 +133,30 @@ public void testGetWorkunitsAfterWatermark() throws Exception { Table table1 = this.hiveMetastoreTestUtils.getLocalMetastoreClient().getTable(dbName, TEST_TABLE_1); + // Denote watermark to have a past created timestamp, so that the watermark workunit gets generated + // This is so that the test is reliable across different operating systems and not flaky to System timing differences previousWorkUnitStates.add(ConversionHiveTestUtils.createWus(dbName, TEST_TABLE_1, - TimeUnit.MILLISECONDS.convert(table1.getCreateTime(), TimeUnit.SECONDS))); + TimeUnit.MILLISECONDS.convert(table1.getCreateTime(), TimeUnit.SECONDS)-100)); SourceState testState = new SourceState(getTestState(dbName), previousWorkUnitStates); testState.setProp(HiveSource.HIVE_SOURCE_WATERMARKER_FACTORY_CLASS_KEY, TableLevelWatermarker.Factory.class.getName()); List workUnits = this.hiveSource.getWorkunits(testState); - Assert.assertEquals(workUnits.size(), 1); + Assert.assertEquals(workUnits.size(), 2); WorkUnit wu = workUnits.get(0); HiveWorkUnit hwu = new HiveWorkUnit(wu); Assert.assertEquals(hwu.getHiveDataset().getDbAndTable().getDb(), dbName); - Assert.assertEquals(hwu.getHiveDataset().getDbAndTable().getTable(), TEST_TABLE_2); + Assert.assertEquals(hwu.getHiveDataset().getDbAndTable().getTable(), TEST_TABLE_1); + + WorkUnit wu2 = workUnits.get(1); + + HiveWorkUnit hwu2 = new HiveWorkUnit(wu2); + + Assert.assertEquals(hwu2.getHiveDataset().getDbAndTable().getDb(), dbName); + Assert.assertEquals(hwu2.getHiveDataset().getDbAndTable().getTable(), TEST_TABLE_2); } @Test