diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/TestTreeNode.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/TestTreeNode.java index adee0321c6..ef490c143d 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/TestTreeNode.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/TestTreeNode.java @@ -19,16 +19,16 @@ package org.apache.amoro; import org.apache.amoro.data.DataTreeNode; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; public class TestTreeNode { - @Test(expected = IllegalArgumentException.class) + @Test public void testParentNodeIllegal() { - DataTreeNode.of(0, 0).parent(); + Assertions.assertThrows(IllegalArgumentException.class, () -> DataTreeNode.of(0, 0).parent()); } @ParameterizedTest @@ -40,12 +40,12 @@ public void testParentNode(int level, int index) { @ParameterizedTest @CsvSource({"0, 0, 1", "1, 0, 2", "1, 1, 3", "3, 0, 4", "3, 3, 7", "7, 5, 13", "7, 3, 11"}) public void testNodeId(int mask, int index, int expectedId) { - Assert.assertEquals(expectedId, DataTreeNode.of(mask, index).getId()); - Assert.assertEquals(DataTreeNode.of(mask, index), DataTreeNode.ofId(expectedId)); + Assertions.assertEquals(expectedId, DataTreeNode.of(mask, index).getId()); + Assertions.assertEquals(DataTreeNode.of(mask, index), DataTreeNode.ofId(expectedId)); } private void assertParentNode(DataTreeNode node) { - Assert.assertEquals(node, node.left().parent()); - Assert.assertEquals(node, node.right().parent()); + Assertions.assertEquals(node, node.left().parent()); + Assertions.assertEquals(node, node.right().parent()); } } diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/TestUnifiedCatalog.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/TestUnifiedCatalog.java index 489eb8b479..ca641b516e 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/TestUnifiedCatalog.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/TestUnifiedCatalog.java @@ -23,53 +23,58 @@ import org.apache.amoro.shade.guava32.com.google.common.collect.Lists; import org.apache.amoro.shade.guava32.com.google.common.collect.Maps; import org.apache.amoro.table.TableMetaStore; -import org.junit.Assert; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.nio.file.Path; +import java.util.stream.Stream; -@RunWith(Parameterized.class) public class TestUnifiedCatalog { - @ClassRule public static TestAms testAms = new TestAms(); + private static final TestAms testAms = new TestAms(); - @Rule public TemporaryFolder warehouse = new TemporaryFolder(); + @TempDir public Path warehouse; - @Parameterized.Parameters - public static Object[] parameters() { - return new Object[] { - TestedCatalogs.hadoopCatalog(TableFormat.ICEBERG), - }; + @BeforeAll + public static void startTestAms() throws Exception { + testAms.before(); } - private final CatalogTestHelper testedCatalog; - private CatalogMeta meta; + @AfterAll + public static void stopTestAms() { + testAms.after(); + } - public TestUnifiedCatalog(CatalogTestHelper testedCatalog) { - this.testedCatalog = testedCatalog; + public static Stream parameters() { + return Stream.of(Arguments.of(TestedCatalogs.hadoopCatalog(TableFormat.ICEBERG))); } - @Before - public void setupCatalogMeta() { - meta = testedCatalog.buildCatalogMeta(warehouse.getRoot().getAbsolutePath()); + private CatalogMeta setupCatalogMeta(CatalogTestHelper testedCatalog) { + CatalogMeta meta = testedCatalog.buildCatalogMeta(warehouse.toFile().getAbsolutePath()); testAms.getAmsHandler().dropCatalog(meta.getCatalogName()); testAms.getAmsHandler().createCatalog(meta); + return meta; } - @Test - public void testCatalogLoader() { + @ParameterizedTest + @MethodSource("parameters") + public void testCatalogLoader(CatalogTestHelper testedCatalog) { + CatalogMeta meta = setupCatalogMeta(testedCatalog); UnifiedCatalog unifiedCatalog = UnifiedCatalogLoader.loadUnifiedCatalog( testAms.getServerUrl(), meta.getCatalogName(), Maps.newHashMap()); validateUnifiedCatalog(unifiedCatalog); } - @Test - public void testCreateUnifiedCatalog() { + @ParameterizedTest + @MethodSource("parameters") + public void testCreateUnifiedCatalog(CatalogTestHelper testedCatalog) { + CatalogMeta meta = setupCatalogMeta(testedCatalog); UnifiedCatalog unifiedCatalog = new CommonUnifiedCatalog( meta.getCatalogName(), @@ -80,13 +85,14 @@ public void testCreateUnifiedCatalog() { } private void validateUnifiedCatalog(UnifiedCatalog unifiedCatalog) { - Assert.assertNotNull(unifiedCatalog); - Assert.assertEquals(CommonUnifiedCatalog.class.getName(), unifiedCatalog.getClass().getName()); + Assertions.assertNotNull(unifiedCatalog); + Assertions.assertEquals( + CommonUnifiedCatalog.class.getName(), unifiedCatalog.getClass().getName()); unifiedCatalog.createDatabase(TableTestHelper.TEST_DB_NAME); - Assert.assertEquals( + Assertions.assertEquals( Lists.newArrayList(TableTestHelper.TEST_DB_NAME), unifiedCatalog.listDatabases()); - Assert.assertEquals(0, unifiedCatalog.listTables(TableTestHelper.TEST_DB_NAME).size()); + Assertions.assertEquals(0, unifiedCatalog.listTables(TableTestHelper.TEST_DB_NAME).size()); unifiedCatalog.dropDatabase(TableTestHelper.TEST_DB_NAME); unifiedCatalog.refresh(); } diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/catalog/CatalogTestBase.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/catalog/CatalogTestBase.java index 1ccbc34960..9deee4184f 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/catalog/CatalogTestBase.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/catalog/CatalogTestBase.java @@ -31,30 +31,99 @@ import org.junit.Before; import org.junit.ClassRule; import org.junit.Rule; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.rules.TemporaryFolder; import java.io.IOException; +/** + * Dual-mode base class supporting both JUnit 4 and JUnit 5 subclasses. + * + *

JUnit 4 path (used by cross-module {@code @RunWith(Parameterized.class)} subclasses in + * amoro-format-mixed-hive and amoro-ams that have not yet migrated): the legacy {@link + * #CatalogTestBase(CatalogTestHelper)} constructor is invoked by the parameterized runner, the + * {@code @ClassRule} {@link #TEST_AMS} starts the mock AMS, the {@code @Rule} {@link #temp} manages + * a temporary folder, and the {@code @Before} {@link #setupCatalog()} / {@code @After} {@link + * #dropCatalog()} fire automatically. + * + *

JUnit 5 path (used by in-module subclasses migrated as part of #4203): the no-arg constructor + * is used, {@link #startTestAms()} / {@link #stopTestAms()} drive the mock AMS via + * {@code @BeforeAll} / {@code @AfterAll}, and subclasses explicitly call {@link + * #setupCatalog(CatalogTestHelper)} from their {@code @ParameterizedTest} method bodies (or from a + * {@code @BeforeEach} when the helper is fixed). The {@link #temp} field is initialised manually + * inside {@link #setupCatalog(CatalogTestHelper)} when null because JUnit 4 {@code @Rule} does not + * fire under the JUnit Platform engine. + * + *

This dual plumbing is removed in the closing PR of #4203 once all cross-module subclasses + * migrate to JUnit 5. + */ public abstract class CatalogTestBase { @ClassRule public static TestAms TEST_AMS = new TestAms(); - private final CatalogTestHelper testHelper; + @Rule public TemporaryFolder temp = new TemporaryFolder(); + + protected CatalogTestHelper testHelper; private UnifiedCatalog unifiedCatalog; private MixedFormatCatalog mixedFormatCatalog; private CatalogMeta catalogMeta; private org.apache.iceberg.catalog.Catalog icebergCatalog; + private boolean tempInitializedByJunit5; + /** JUnit 4 constructor — kept for cross-module {@code @RunWith(Parameterized.class)} callers. */ public CatalogTestBase(CatalogTestHelper testHelper) { this.testHelper = testHelper; } + /** JUnit 5 constructor — used by in-module migrated subclasses. */ + public CatalogTestBase() {} + + @BeforeAll + public static void startTestAms() throws Exception { + TEST_AMS.before(); + } + + @AfterAll + public static void stopTestAms() { + TEST_AMS.after(); + } + public static MockAmoroManagementServer.AmsHandler getAmsHandler() { return TEST_AMS.getAmsHandler(); } + /** JUnit 4 lifecycle — fires when subclass uses {@code @RunWith(Parameterized)}. */ @Before public void setupCatalog() throws IOException { + if (testHelper != null) { + doSetupCatalog(); + } + } + + /** + * JUnit 5 entry point — invoke from {@code @ParameterizedTest} method body or + * {@code @BeforeEach}. + */ + protected void setupCatalog(CatalogTestHelper helper) throws IOException { + this.testHelper = helper; + // JUnit 4 @Rule does not fire under JUnit Platform; manually create the temp folder. + if (!tempInitializedByJunit5) { + if (temp == null) { + temp = new TemporaryFolder(); + } + try { + temp.create(); + } catch (IOException e) { + throw e; + } + tempInitializedByJunit5 = true; + } + doSetupCatalog(); + } + + private void doSetupCatalog() throws IOException { String baseDir = temp.newFolder().getPath(); if (!SystemUtils.IS_OS_UNIX) { baseDir = "file:/" + temp.newFolder().getPath().replace("\\", "/"); @@ -65,11 +134,17 @@ public void setupCatalog() throws IOException { } @After + @AfterEach public void dropCatalog() { if (catalogMeta != null) { getAmsHandler().dropCatalog(catalogMeta.getCatalogName()); mixedFormatCatalog = null; } + if (tempInitializedByJunit5 && temp != null) { + temp.delete(); + temp = null; + tempInitializedByJunit5 = false; + } } protected MixedFormatCatalog getMixedFormatCatalog() { diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/catalog/TableTestBase.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/catalog/TableTestBase.java index ae433b2089..eabcd6c1f9 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/catalog/TableTestBase.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/catalog/TableTestBase.java @@ -28,20 +28,55 @@ import org.apache.amoro.utils.MixedTableUtil; import org.junit.After; import org.junit.Before; +import org.junit.jupiter.api.AfterEach; +import java.io.IOException; + +/** + * Dual-mode base class supporting both JUnit 4 and JUnit 5 subclasses. See {@link CatalogTestBase} + * for the full dual-mode rationale; the same constraints apply here. Cross-module + * {@code @RunWith(Parameterized.class)} subclasses keep the legacy two-arg constructor and rely on + * the {@code @Before} / {@code @After} hooks. In-module JUnit 5 subclasses use the no-arg + * constructor and explicitly call {@link #setupTable(CatalogTestHelper, TableTestHelper)} from + * their {@code @ParameterizedTest} method body or {@code @BeforeEach}. + */ public abstract class TableTestBase extends CatalogTestBase { - private final TableTestHelper tableTestHelper; + protected TableTestHelper tableTestHelper; private MixedTable mixedTable; private TableMetaStore tableMetaStore; + /** JUnit 4 constructor — kept for cross-module {@code @RunWith(Parameterized.class)} callers. */ public TableTestBase(CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) { super(catalogTestHelper); this.tableTestHelper = tableTestHelper; } + /** JUnit 5 constructor — used by in-module migrated subclasses. */ + public TableTestBase() { + super(); + } + + /** JUnit 4 lifecycle — fires when subclass uses {@code @RunWith(Parameterized)}. */ @Before public void setupTable() { + if (tableTestHelper != null) { + doSetupTable(); + } + } + + /** + * JUnit 5 entry point — invoke from {@code @ParameterizedTest} method body or + * {@code @BeforeEach}. + */ + protected void setupTable(CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) + throws IOException { + setupCatalog(catalogTestHelper); + this.tableTestHelper = tableTestHelper; + doSetupTable(); + } + + private void doSetupTable() { this.tableMetaStore = CatalogUtil.buildMetaStore(getCatalogMeta()); getUnifiedCatalog().createDatabase(TableTestHelper.TEST_DB_NAME); @@ -83,7 +118,11 @@ private void createIcebergFormatTable() { } @After + @AfterEach public void dropTable() { + if (tableTestHelper == null) { + return; + } getUnifiedCatalog() .dropTable(tableTestHelper.id().getDatabase(), tableTestHelper.id().getTableName(), true); try { diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/catalog/TestCatalogLoader.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/catalog/TestCatalogLoader.java index b471492336..273390e6f6 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/catalog/TestCatalogLoader.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/catalog/TestCatalogLoader.java @@ -26,16 +26,27 @@ import org.apache.amoro.mixed.MixedFormatCatalog; import org.apache.amoro.properties.CatalogMetaProperties; import org.apache.amoro.shade.guava32.com.google.common.collect.Maps; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import java.util.Map; public class TestCatalogLoader { private static final String TEST_CATALOG_NAME = "test"; - @ClassRule public static TestAms TEST_AMS = new TestAms(); + private static final TestAms TEST_AMS = new TestAms(); + + @BeforeAll + public static void startTestAms() throws Exception { + TEST_AMS.before(); + } + + @AfterAll + public static void stopTestAms() { + TEST_AMS.after(); + } @Test public void testLoadMixedIcebergCatalog() { @@ -49,17 +60,18 @@ public void testLoadMixedIcebergCatalog() { TableFormat.MIXED_ICEBERG); TEST_AMS.getAmsHandler().createCatalog(catalogMeta); MixedFormatCatalog loadCatalog = CatalogLoader.load(getCatalogUrl(TEST_CATALOG_NAME)); - Assert.assertEquals(TEST_CATALOG_NAME, loadCatalog.name()); - Assert.assertEquals(BasicMixedIcebergCatalog.class.getName(), loadCatalog.getClass().getName()); + Assertions.assertEquals(TEST_CATALOG_NAME, loadCatalog.name()); + Assertions.assertEquals( + BasicMixedIcebergCatalog.class.getName(), loadCatalog.getClass().getName()); TEST_AMS.getAmsHandler().dropCatalog(TEST_CATALOG_NAME); } @Test public void testLoadNotExistedCatalog() { - Assert.assertThrows( - "catalog not found, please check catalog name", + Assertions.assertThrows( IllegalArgumentException.class, - () -> CatalogLoader.load(getCatalogUrl(TEST_CATALOG_NAME))); + () -> CatalogLoader.load(getCatalogUrl(TEST_CATALOG_NAME)), + "catalog not found, please check catalog name"); } private String getCatalogUrl(String catalogName) { diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/data/TestDefaultKeyedFile.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/data/TestDefaultKeyedFile.java index b64d148f9c..83cae94edf 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/data/TestDefaultKeyedFile.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/data/TestDefaultKeyedFile.java @@ -26,15 +26,18 @@ import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; import org.apache.iceberg.DataFile; import org.apache.iceberg.data.Record; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import java.io.IOException; import java.util.List; public class TestDefaultKeyedFile extends TableTestBase { - public TestDefaultKeyedFile() { - super( + @BeforeEach + public void setUp() throws IOException { + setupTable( new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), new BasicTableTestHelper(true, true)); } @@ -46,12 +49,12 @@ public void testDefaultKeyedFile() { MixedDataTestHelpers.writeChangeStore( getMixedTable().asKeyedTable(), txId, ChangeAction.INSERT, writeRecords(), false); - Assert.assertEquals(1, writeFiles.size()); + Assertions.assertEquals(1, writeFiles.size()); DefaultKeyedFile defaultKeyedFile = DefaultKeyedFile.parseChange(writeFiles.get(0)); - Assert.assertEquals(DataFileType.INSERT_FILE, defaultKeyedFile.type()); - Assert.assertEquals(3, defaultKeyedFile.node().mask()); - Assert.assertEquals(0, defaultKeyedFile.node().index()); - Assert.assertEquals(txId, defaultKeyedFile.transactionId()); + Assertions.assertEquals(DataFileType.INSERT_FILE, defaultKeyedFile.type()); + Assertions.assertEquals(3, defaultKeyedFile.node().mask()); + Assertions.assertEquals(0, defaultKeyedFile.node().index()); + Assertions.assertEquals(txId, defaultKeyedFile.transactionId()); } private List writeRecords() { diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/data/TestUpsertPushDown.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/data/TestUpsertPushDown.java index 0895e5271b..760e7034de 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/data/TestUpsertPushDown.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/data/TestUpsertPushDown.java @@ -30,51 +30,50 @@ import org.apache.iceberg.data.Record; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; -import org.junit.Assert; -import org.junit.Assume; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import java.util.stream.Stream; -@RunWith(Parameterized.class) public class TestUpsertPushDown extends TableTestBase { - public TestUpsertPushDown(PartitionSpec partitionSpec) { - super( - new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), - new BasicTableTestHelper( - BasicTableTestHelper.TABLE_SCHEMA, - BasicTableTestHelper.PRIMARY_KEY_SPEC, - partitionSpec, - buildTableProperties())); - } - private static Map buildTableProperties() { Map properties = Maps.newHashMap(); properties.put(TableProperties.UPSERT_ENABLED, "true"); return properties; } - @Parameterized.Parameters(name = "spec = {0}") - public static Object[] parameters() { - return new Object[] { - PartitionSpec.unpartitioned(), - BasicTableTestHelper.SPEC, - PartitionSpec.builderFor(BasicTableTestHelper.TABLE_SCHEMA) - .day("op_time") - .identity("ts") - .build() - }; + public static Stream parameters() { + return Stream.of( + Arguments.of(PartitionSpec.unpartitioned()), + Arguments.of(BasicTableTestHelper.SPEC), + Arguments.of( + PartitionSpec.builderFor(BasicTableTestHelper.TABLE_SCHEMA) + .day("op_time") + .identity("ts") + .build())); + } + + private void prepareTable(PartitionSpec partitionSpec) throws IOException { + setupTable( + new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), + new BasicTableTestHelper( + BasicTableTestHelper.TABLE_SCHEMA, + BasicTableTestHelper.PRIMARY_KEY_SPEC, + partitionSpec, + buildTableProperties())); + initChangeStoreData(); } - @Before - public void initChangeStoreData() { + private void initChangeStoreData() { MixedDataTestHelpers.writeAndCommitChangeStore( getMixedTable().asKeyedTable(), 1L, @@ -113,18 +112,23 @@ public void initChangeStoreData() { false); } - @Test - public void testReadKeyedTableWithoutFilter() { + @ParameterizedTest(name = "spec = {0}") + @MethodSource("parameters") + public void testReadKeyedTableWithoutFilter(PartitionSpec partitionSpec) throws IOException { + prepareTable(partitionSpec); List records = MixedDataTestHelpers.readKeyedTable( getMixedTable().asKeyedTable(), Expressions.alwaysTrue()); - Assert.assertEquals(records.size(), 2); - Assert.assertTrue(recordToNameList(records).containsAll(Arrays.asList("aaa", "ccc"))); + Assertions.assertEquals(records.size(), 2); + Assertions.assertTrue(recordToNameList(records).containsAll(Arrays.asList("aaa", "ccc"))); } - @Test - public void testReadKeyedTableWithPartitionAndColumnFilter() { - Assume.assumeTrue(isPartitionedTable()); + @ParameterizedTest(name = "spec = {0}") + @MethodSource("parameters") + public void testReadKeyedTableWithPartitionAndColumnFilter(PartitionSpec partitionSpec) + throws IOException { + prepareTable(partitionSpec); + Assumptions.assumeTrue(isPartitionedTable()); Expression partitionAndColumnFilter = Expressions.and( Expressions.and( @@ -136,13 +140,16 @@ public void testReadKeyedTableWithPartitionAndColumnFilter() { getMixedTable().asKeyedTable(), partitionAndColumnFilter); // Scan from change store only filter partition column expression, so record(name=ccc) is still // returned. - Assert.assertEquals(records.size(), 1); - Assert.assertTrue(recordToNameList(records).contains("ccc")); + Assertions.assertEquals(records.size(), 1); + Assertions.assertTrue(recordToNameList(records).contains("ccc")); } - @Test - public void testReadKeyedTableWithPartitionOrColumnFilter() { - Assume.assumeTrue(isPartitionedTable()); + @ParameterizedTest(name = "spec = {0}") + @MethodSource("parameters") + public void testReadKeyedTableWithPartitionOrColumnFilter(PartitionSpec partitionSpec) + throws IOException { + prepareTable(partitionSpec); + Assumptions.assumeTrue(isPartitionedTable()); Expression partitionOrColumnFilter = Expressions.or( Expressions.and( @@ -152,35 +159,43 @@ public void testReadKeyedTableWithPartitionOrColumnFilter() { List records = MixedDataTestHelpers.readKeyedTable( getMixedTable().asKeyedTable(), partitionOrColumnFilter); - Assert.assertEquals(records.size(), 2); - Assert.assertTrue(recordToNameList(records).containsAll(Arrays.asList("aaa", "ccc"))); + Assertions.assertEquals(records.size(), 2); + Assertions.assertTrue(recordToNameList(records).containsAll(Arrays.asList("aaa", "ccc"))); } - @Test - public void testReadKeyedTableWithPartitionFilter() { - Assume.assumeTrue(isPartitionedTable()); + @ParameterizedTest(name = "spec = {0}") + @MethodSource("parameters") + public void testReadKeyedTableWithPartitionFilter(PartitionSpec partitionSpec) + throws IOException { + prepareTable(partitionSpec); + Assumptions.assumeTrue(isPartitionedTable()); Expression partitionFilter = Expressions.and( Expressions.notNull("op_time"), Expressions.equal("op_time", "2022-01-02T12:00:00")); List records = MixedDataTestHelpers.readKeyedTable(getMixedTable().asKeyedTable(), partitionFilter); - Assert.assertEquals(records.size(), 1); - Assert.assertTrue(recordToNameList(records).contains("ccc")); + Assertions.assertEquals(records.size(), 1); + Assertions.assertTrue(recordToNameList(records).contains("ccc")); } - @Test - public void testReadKeyedTableWithColumnFilter() { + @ParameterizedTest(name = "spec = {0}") + @MethodSource("parameters") + public void testReadKeyedTableWithColumnFilter(PartitionSpec partitionSpec) throws IOException { + prepareTable(partitionSpec); Expression columnFilter = Expressions.and(Expressions.notNull("name"), Expressions.equal("name", "bbb")); List records = MixedDataTestHelpers.readKeyedTable(getMixedTable().asKeyedTable(), columnFilter); - Assert.assertEquals(records.size(), 2); - Assert.assertTrue(recordToNameList(records).containsAll(Arrays.asList("aaa", "ccc"))); + Assertions.assertEquals(records.size(), 2); + Assertions.assertTrue(recordToNameList(records).containsAll(Arrays.asList("aaa", "ccc"))); } - @Test - public void testReadKeyedTableWithGreaterPartitionAndColumnFilter() { - Assume.assumeTrue(isPartitionedTable()); + @ParameterizedTest(name = "spec = {0}") + @MethodSource("parameters") + public void testReadKeyedTableWithGreaterPartitionAndColumnFilter(PartitionSpec partitionSpec) + throws IOException { + prepareTable(partitionSpec); + Assumptions.assumeTrue(isPartitionedTable()); Expression greaterPartitionAndColumnFilter = Expressions.and( Expressions.and( @@ -190,8 +205,8 @@ public void testReadKeyedTableWithGreaterPartitionAndColumnFilter() { List records = MixedDataTestHelpers.readKeyedTable( getMixedTable().asKeyedTable(), greaterPartitionAndColumnFilter); - Assert.assertEquals(records.size(), 1); - Assert.assertTrue(recordToNameList(records).contains("ccc")); + Assertions.assertEquals(records.size(), 1); + Assertions.assertTrue(recordToNameList(records).contains("ccc")); } private List writeRecords(int id, String name, long ts, int day) { diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/formats/iceberg/maintainer/TestAutoCreateIcebergTagAction.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/formats/iceberg/maintainer/TestAutoCreateIcebergTagAction.java index 4eb637edd9..5e1fcebb50 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/formats/iceberg/maintainer/TestAutoCreateIcebergTagAction.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/formats/iceberg/maintainer/TestAutoCreateIcebergTagAction.java @@ -31,9 +31,11 @@ import org.apache.iceberg.SnapshotRef; import org.apache.iceberg.Table; import org.jetbrains.annotations.NotNull; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import java.io.IOException; import java.time.Duration; import java.time.Instant; import java.time.LocalDateTime; @@ -45,8 +47,10 @@ public class TestAutoCreateIcebergTagAction extends TableTestBase { - public TestAutoCreateIcebergTagAction() { - super(new BasicCatalogTestHelper(TableFormat.ICEBERG), new BasicTableTestHelper(false, true)); + @BeforeEach + public void setUp() throws IOException { + setupTable( + new BasicCatalogTestHelper(TableFormat.ICEBERG), new BasicTableTestHelper(false, true)); } @Test @@ -325,7 +329,7 @@ private void testTagTimePeriodHourly( .toInstant() .toEpochMilli(); - Assert.assertEquals(expectedTriggerTime, actualTriggerTime); + Assertions.assertEquals(expectedTriggerTime, actualTriggerTime); } private void testTagTimePeriodDaily( @@ -346,7 +350,7 @@ private void testTagTimePeriodDaily( .toInstant() .toEpochMilli(); - Assert.assertEquals(expectedTriggerTime, actualTriggerTime); + Assertions.assertEquals(expectedTriggerTime, actualTriggerTime); } /** @@ -426,22 +430,23 @@ private String formatDateTime(LocalDateTime localDateTime) { } private void checkNoTag(Table table) { - Assert.assertFalse(table.refs().values().stream().anyMatch(SnapshotRef::isTag)); + Assertions.assertFalse(table.refs().values().stream().anyMatch(SnapshotRef::isTag)); } private void checkTagCount(Table table, int count) { - Assert.assertEquals(count, table.refs().values().stream().filter(SnapshotRef::isTag).count()); + Assertions.assertEquals( + count, table.refs().values().stream().filter(SnapshotRef::isTag).count()); } private void checkTag(Table table, String tagName, Snapshot snapshot) { SnapshotRef snapshotRef = table.refs().get(tagName); - Assert.assertNotNull(snapshotRef); - Assert.assertTrue(snapshotRef.isTag()); - Assert.assertEquals(snapshot.snapshotId(), snapshotRef.snapshotId()); + Assertions.assertNotNull(snapshotRef); + Assertions.assertTrue(snapshotRef.isTag()); + Assertions.assertEquals(snapshot.snapshotId(), snapshotRef.snapshotId()); } private void checkSnapshots(Table table, int count) { - Assert.assertEquals(Iterables.size(table.snapshots()), count); + Assertions.assertEquals(Iterables.size(table.snapshots()), count); } private long waitUntilAfter(long timestampMillis) { diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TableDataTestBase.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TableDataTestBase.java index 806955f0cf..cdec9bf681 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TableDataTestBase.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TableDataTestBase.java @@ -37,6 +37,13 @@ import java.util.Collections; import java.util.List; +/** + * Dual-mode base class supporting both JUnit 4 and JUnit 5 subclasses. JUnit 4 subclasses (still in + * amoro-format-mixed-hive) use the legacy {@code (CatalogTestHelper, TableTestHelper)} constructor + * and rely on the inherited {@code @Before} lifecycle plus this class's own {@code @Before + * initData()}. JUnit 5 subclasses use the no-arg constructor and explicitly call {@link + * #initData()} or {@link #initData(CatalogTestHelper, TableTestHelper)} from a {@code @BeforeEach}. + */ public abstract class TableDataTestBase extends TableTestBase { // 6 records, (id=1),(id=2),(id=3),(id=4),(id=5),(id=6) @@ -45,14 +52,14 @@ public abstract class TableDataTestBase extends TableTestBase { protected DataFile dataFileForPositionDelete; protected DeleteFile deleteFileOfPositionDelete; + /** JUnit 4 constructor — kept for cross-module {@code @RunWith(Parameterized.class)} callers. */ public TableDataTestBase(CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) { super(catalogTestHelper, tableTestHelper); } + /** JUnit 4 default constructor — used by JUnit 4 subclasses with fixed helpers. */ public TableDataTestBase() { - this( - new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), - new BasicTableTestHelper(true, true)); + super(); } protected List baseRecords(List records) { @@ -78,8 +85,31 @@ protected List changeDeleteRecords(List records) { return builder.build(); } + /** + * JUnit 4 lifecycle — fires for {@code @RunWith(Parameterized)} subclasses; tableTestHelper has + * already been set by the constructor and parent {@code @Before}. JUnit 5 subclasses must invoke + * {@link #initData()} explicitly from their {@code @BeforeEach}. + */ @Before public void initData() throws IOException { + if (tableTestHelper == null) { + // JUnit 5 path with no helper yet — fall back to default helper combination. + initData( + new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), + new BasicTableTestHelper(true, true)); + return; + } + populateData(); + } + + /** JUnit 5 entry point — call from a {@code @BeforeEach} or {@code @ParameterizedTest} body. */ + protected void initData(CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) + throws IOException { + setupTable(catalogTestHelper, tableTestHelper); + populateData(); + } + + private void populateData() throws IOException { allRecords = Lists.newArrayListWithCapacity(6); allRecords.add(tableTestHelper().generateTestRecord(1, "john", 0, "2022-01-01T12:00:00")); allRecords.add(tableTestHelper().generateTestRecord(2, "lily", 0, "2022-01-02T12:00:00")); diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestBasicTableTrashManager.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestBasicTableTrashManager.java index 0f7fb4701c..61eccd1064 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestBasicTableTrashManager.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestBasicTableTrashManager.java @@ -30,60 +30,65 @@ import org.apache.amoro.utils.TableFileUtil; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.OutputFile; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneId; +import java.util.stream.Stream; -@RunWith(Parameterized.class) public class TestBasicTableTrashManager extends TableTestBase { - public TestBasicTableTrashManager(boolean keyedTable, boolean partitionedTable) { - super( - new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), - new BasicTableTestHelper(keyedTable, partitionedTable)); - } + @TempDir public Path tempTrashLocation; - @Parameterized.Parameters(name = "keyedTable = {0}, partitionedTable = {1}") - public static Object[][] parameters() { - return new Object[][] { - {true, true}, - {true, false}, - {false, true}, - {false, false} - }; + public static Stream parameters() { + return Stream.of( + Arguments.of(true, true), + Arguments.of(true, false), + Arguments.of(false, true), + Arguments.of(false, false)); } - @Rule public TemporaryFolder tempTrashLocation = new TemporaryFolder(); + private void prepare(boolean keyedTable, boolean partitionedTable) throws IOException { + setupTable( + new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), + new BasicTableTestHelper(keyedTable, partitionedTable)); + } - @Test - public void testGenerateFileLocationInTrash() { + @ParameterizedTest(name = "keyedTable = {0}, partitionedTable = {1}") + @MethodSource("parameters") + public void testGenerateFileLocationInTrash(boolean keyedTable, boolean partitionedTable) + throws IOException { + prepare(keyedTable, partitionedTable); String relativeFileLocation = getRelativeFileLocation("/tmp/table", "/tmp/table/change/file1"); - Assert.assertEquals("change/file1", relativeFileLocation); + Assertions.assertEquals("change/file1", relativeFileLocation); relativeFileLocation = getRelativeFileLocation("/tmp/table/", "/tmp/table/change/file1"); - Assert.assertEquals("change/file1", relativeFileLocation); + Assertions.assertEquals("change/file1", relativeFileLocation); relativeFileLocation = getRelativeFileLocation( "hdfs://hz11-trino-arctic-0.jd.163.org:8020/user/warehouse/", "/user/warehouse/change/file1"); - Assert.assertEquals("change/file1", relativeFileLocation); + Assertions.assertEquals("change/file1", relativeFileLocation); LocalDateTime localDateTime = LocalDateTime.of(2023, 2, 2, 1, 1); long toEpochMilli = localDateTime.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); String locationInTrash = generateFileLocationInTrash("change/file1", "/tmp/table/.trash", toEpochMilli); - Assert.assertEquals("/tmp/table/.trash/20230202/change/file1", locationInTrash); + Assertions.assertEquals("/tmp/table/.trash/20230202/change/file1", locationInTrash); } - @Test - public void testDeleteAndRestore() throws IOException { + @ParameterizedTest(name = "keyedTable = {0}, partitionedTable = {1}") + @MethodSource("parameters") + public void testDeleteAndRestore(boolean keyedTable, boolean partitionedTable) + throws IOException { + prepare(keyedTable, partitionedTable); String tableRootLocation = getMixedTable().location(); TableTrashManager tableTrashManager = build(); String trashLocation = tableTrashManager.getTrashLocation(); @@ -92,25 +97,28 @@ public void testDeleteAndRestore() throws IOException { String path = createFile(getMixedTable().io(), fullLocation(tableRootLocation, relativeFilePath)); - Assert.assertFalse(tableTrashManager.fileExistInTrash(path)); - Assert.assertFalse(tableTrashManager.restoreFileFromTrash(path)); + Assertions.assertFalse(tableTrashManager.fileExistInTrash(path)); + Assertions.assertFalse(tableTrashManager.restoreFileFromTrash(path)); long now = System.currentTimeMillis(); tableTrashManager.moveFileToTrash(path); String fileLocationInTrash = generateFileLocationInTrash(relativeFilePath, trashLocation, now); - Assert.assertFalse(getMixedTable().io().exists(path)); - Assert.assertTrue(getMixedTable().io().exists(fileLocationInTrash)); + Assertions.assertFalse(getMixedTable().io().exists(path)); + Assertions.assertTrue(getMixedTable().io().exists(fileLocationInTrash)); - Assert.assertTrue(tableTrashManager.fileExistInTrash(path)); - Assert.assertTrue(tableTrashManager.restoreFileFromTrash(path)); + Assertions.assertTrue(tableTrashManager.fileExistInTrash(path)); + Assertions.assertTrue(tableTrashManager.restoreFileFromTrash(path)); - Assert.assertTrue(getMixedTable().io().exists(path)); - Assert.assertFalse(getMixedTable().io().exists(fileLocationInTrash)); + Assertions.assertTrue(getMixedTable().io().exists(path)); + Assertions.assertFalse(getMixedTable().io().exists(fileLocationInTrash)); } - @Test - public void testMoveAndOverwrite() throws IOException { + @ParameterizedTest(name = "keyedTable = {0}, partitionedTable = {1}") + @MethodSource("parameters") + public void testMoveAndOverwrite(boolean keyedTable, boolean partitionedTable) + throws IOException { + prepare(keyedTable, partitionedTable); String tableRootLocation = getMixedTable().location(); TableTrashManager tableTrashManager = build(); @@ -119,14 +127,16 @@ public void testMoveAndOverwrite() throws IOException { createFile(getMixedTable().io(), fullLocation(tableRootLocation, relativeFilePath)); tableTrashManager.moveFileToTrash(path); - Assert.assertTrue(tableTrashManager.fileExistInTrash(path)); + Assertions.assertTrue(tableTrashManager.fileExistInTrash(path)); createFile(getMixedTable().io(), fullLocation(tableRootLocation, relativeFilePath)); tableTrashManager.moveFileToTrash(path); - Assert.assertTrue(tableTrashManager.fileExistInTrash(path)); + Assertions.assertTrue(tableTrashManager.fileExistInTrash(path)); } - @Test - public void testDeleteDirectory() throws IOException { + @ParameterizedTest(name = "keyedTable = {0}, partitionedTable = {1}") + @MethodSource("parameters") + public void testDeleteDirectory(boolean keyedTable, boolean partitionedTable) throws IOException { + prepare(keyedTable, partitionedTable); String tableRootLocation = getMixedTable().location(); TableTrashManager tableTrashManager = build(); String trashLocation = tableTrashManager.getTrashLocation(); @@ -137,21 +147,24 @@ public void testDeleteDirectory() throws IOException { String directory = TableFileUtil.getFileDir(path); long now = System.currentTimeMillis(); IllegalArgumentException illegalArgumentException = - Assert.assertThrows( - "should not successfully move a directory to trash", + Assertions.assertThrows( IllegalArgumentException.class, - () -> tableTrashManager.moveFileToTrash(directory)); - Assert.assertTrue(illegalArgumentException.getMessage().contains("directory")); + () -> tableTrashManager.moveFileToTrash(directory), + "should not successfully move a directory to trash"); + Assertions.assertTrue(illegalArgumentException.getMessage().contains("directory")); String relativeDirectory = getRelativeFileLocation(tableRootLocation, directory); String directoryLocationInTrash = generateFileLocationInTrash(relativeDirectory, trashLocation, now); - Assert.assertTrue(getMixedTable().io().exists(directory)); - Assert.assertFalse(getMixedTable().io().exists(directoryLocationInTrash)); + Assertions.assertTrue(getMixedTable().io().exists(directory)); + Assertions.assertFalse(getMixedTable().io().exists(directoryLocationInTrash)); } - @Test - public void testRestoreDirectory() throws IOException { + @ParameterizedTest(name = "keyedTable = {0}, partitionedTable = {1}") + @MethodSource("parameters") + public void testRestoreDirectory(boolean keyedTable, boolean partitionedTable) + throws IOException { + prepare(keyedTable, partitionedTable); String tableRootLocation = getMixedTable().location(); TableTrashManager tableTrashManager = build(); String trashLocation = tableTrashManager.getTrashLocation(); @@ -163,15 +176,17 @@ public void testRestoreDirectory() throws IOException { tableTrashManager.moveFileToTrash(path); String fileLocationInTrash = generateFileLocationInTrash(relativeFilePath, trashLocation, now); - Assert.assertFalse(getMixedTable().io().exists(path)); - Assert.assertTrue(getMixedTable().io().exists(fileLocationInTrash)); + Assertions.assertFalse(getMixedTable().io().exists(path)); + Assertions.assertTrue(getMixedTable().io().exists(fileLocationInTrash)); - Assert.assertFalse(getMixedTable().io().exists(path)); - Assert.assertTrue(getMixedTable().io().exists(fileLocationInTrash)); + Assertions.assertFalse(getMixedTable().io().exists(path)); + Assertions.assertTrue(getMixedTable().io().exists(fileLocationInTrash)); } - @Test - public void testCleanFiles() throws IOException { + @ParameterizedTest(name = "keyedTable = {0}, partitionedTable = {1}") + @MethodSource("parameters") + public void testCleanFiles(boolean keyedTable, boolean partitionedTable) throws IOException { + prepare(keyedTable, partitionedTable); String tableRootLocation = getMixedTable().location(); BasicTableTrashManager tableTrashManager = ((BasicTableTrashManager) build()); String trashLocation = tableTrashManager.getTrashLocation(); @@ -219,34 +234,41 @@ public void testCleanFiles() throws IOException { createFile(getMixedTable().io(), file5Day3); createFile(getMixedTable().io(), illegalFile); - Assert.assertTrue(tableTrashManager.fileExistInTrash(file1)); - Assert.assertTrue(tableTrashManager.fileExistInTrash(file2)); - Assert.assertTrue(tableTrashManager.fileExistInTrash(file3)); - Assert.assertTrue(tableTrashManager.fileExistInTrash(file4)); - Assert.assertTrue(tableTrashManager.fileExistInTrash(file5)); - Assert.assertFalse(tableTrashManager.fileExistInTrash(illegalFile)); + Assertions.assertTrue(tableTrashManager.fileExistInTrash(file1)); + Assertions.assertTrue(tableTrashManager.fileExistInTrash(file2)); + Assertions.assertTrue(tableTrashManager.fileExistInTrash(file3)); + Assertions.assertTrue(tableTrashManager.fileExistInTrash(file4)); + Assertions.assertTrue(tableTrashManager.fileExistInTrash(file5)); + Assertions.assertFalse(tableTrashManager.fileExistInTrash(illegalFile)); tableTrashManager.cleanFiles(LocalDate.of(2023, 2, 22)); - Assert.assertFalse(tableTrashManager.fileExistInTrash(file1)); - Assert.assertFalse(tableTrashManager.fileExistInTrash(file2)); - Assert.assertFalse(tableTrashManager.fileExistInTrash(file3)); - Assert.assertFalse(tableTrashManager.fileExistInTrash(file4)); - Assert.assertTrue(tableTrashManager.fileExistInTrash(file5)); - Assert.assertFalse(tableTrashManager.fileExistInTrash(illegalFile)); - - Assert.assertFalse(getMixedTable().io().exists(file1Day1)); - Assert.assertFalse(getMixedTable().io().exists(file2Day1)); - Assert.assertFalse(getMixedTable().io().exists(file3Day1)); - Assert.assertFalse(getMixedTable().io().exists(file4Day2)); - Assert.assertTrue(getMixedTable().io().exists(file5Day3)); - Assert.assertTrue(getMixedTable().io().exists(illegalFile)); + Assertions.assertFalse(tableTrashManager.fileExistInTrash(file1)); + Assertions.assertFalse(tableTrashManager.fileExistInTrash(file2)); + Assertions.assertFalse(tableTrashManager.fileExistInTrash(file3)); + Assertions.assertFalse(tableTrashManager.fileExistInTrash(file4)); + Assertions.assertTrue(tableTrashManager.fileExistInTrash(file5)); + Assertions.assertFalse(tableTrashManager.fileExistInTrash(illegalFile)); + + Assertions.assertFalse(getMixedTable().io().exists(file1Day1)); + Assertions.assertFalse(getMixedTable().io().exists(file2Day1)); + Assertions.assertFalse(getMixedTable().io().exists(file3Day1)); + Assertions.assertFalse(getMixedTable().io().exists(file4Day2)); + Assertions.assertTrue(getMixedTable().io().exists(file5Day3)); + Assertions.assertTrue(getMixedTable().io().exists(illegalFile)); } - @Test - public void testDeleteTrashLocation() throws IOException { + @ParameterizedTest(name = "keyedTable = {0}, partitionedTable = {1}") + @MethodSource("parameters") + public void testDeleteTrashLocation(boolean keyedTable, boolean partitionedTable) + throws IOException { + prepare(keyedTable, partitionedTable); String tableRootLocation = getMixedTable().location(); - String customTrashLocation = tempTrashLocation.newFolder().getPath().replace('\\', '/'); + String customTrashLocation = + Files.createTempDirectory(tempTrashLocation, "trash-") + .toFile() + .getPath() + .replace('\\', '/'); getMixedTable() .updateProperties() @@ -256,17 +278,17 @@ public void testDeleteTrashLocation() throws IOException { createFile(getMixedTable().io(), file1); TableTrashManager tableTrashManager = build(); tableTrashManager.moveFileToTrash(file1); - Assert.assertTrue(tableTrashManager.fileExistInTrash(file1)); + Assertions.assertTrue(tableTrashManager.fileExistInTrash(file1)); String trashParentLocation = TableTrashManagers.getTrashParentLocation(getMixedTable().id(), customTrashLocation); getMixedFormatCatalog().dropTable(getMixedTable().id(), true); - Assert.assertFalse(getMixedTable().io().exists(trashParentLocation)); - Assert.assertFalse(tableTrashManager.fileExistInTrash(file1)); + Assertions.assertFalse(getMixedTable().io().exists(trashParentLocation)); + Assertions.assertFalse(tableTrashManager.fileExistInTrash(file1)); } private TableTrashManager build() { MixedTable table = getMixedTable(); - Assert.assertTrue(table.io() instanceof AuthenticatedHadoopFileIO); + Assertions.assertTrue(table.io() instanceof AuthenticatedHadoopFileIO); return TableTrashManagers.build( table.id(), getMixedTable().location(), diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestCombinedDeleteFilterStructProjection.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestCombinedDeleteFilterStructProjection.java index 73d2cbd856..582b165e13 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestCombinedDeleteFilterStructProjection.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestCombinedDeleteFilterStructProjection.java @@ -43,11 +43,10 @@ import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.OutputFileFactory; import org.apache.iceberg.types.TypeUtil; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; import java.util.ArrayList; @@ -56,6 +55,7 @@ import java.util.List; import java.util.Map; import java.util.stream.IntStream; +import java.util.stream.Stream; /** * Tests for StructProjection hoisting in CombinedDeleteFilter.initializeBloomFilter(). @@ -64,22 +64,21 @@ * projection object via .wrap()) produces correct bloom filter initialization and equality-delete * filtering results, including the multiple-delete-schema scenario. */ -@RunWith(Parameterized.class) public class TestCombinedDeleteFilterStructProjection extends TableTestBase { - private final FileFormat fileFormat; + private FileFormat fileFormat; - @Parameterized.Parameters(name = "fileFormat = {0}") - public static Object[][] parameters() { - return new Object[][] {{FileFormat.PARQUET}, {FileFormat.AVRO}, {FileFormat.ORC}}; - } + /** + * Lower the bloom-filter threshold so the filter is activated with a small dataset (< 3 data + * records) while still having > threshold eq-delete records. + */ + private static final long BLOOM_TRIGGER = 2L; - public TestCombinedDeleteFilterStructProjection(FileFormat fileFormat) { - super( - new BasicCatalogTestHelper(TableFormat.ICEBERG), - new BasicTableTestHelper(false, false, buildTableProperties(fileFormat))); - this.fileFormat = fileFormat; - System.setProperty(DeleteCache.DELETE_CACHE_ENABLED, "false"); + public static Stream parameters() { + return Stream.of( + Arguments.of(FileFormat.PARQUET), + Arguments.of(FileFormat.AVRO), + Arguments.of(FileFormat.ORC)); } private static Map buildTableProperties(FileFormat fileFormat) { @@ -90,14 +89,12 @@ private static Map buildTableProperties(FileFormat fileFormat) { return props; } - /** - * Lower the bloom-filter threshold so the filter is activated with a small dataset (< 3 data - * records) while still having > threshold eq-delete records. - */ - private static final long BLOOM_TRIGGER = 2L; - - @Before - public void resetBloomFilterThreshold() { + private void prepare(FileFormat fileFormat) throws IOException { + setupTable( + new BasicCatalogTestHelper(TableFormat.ICEBERG), + new BasicTableTestHelper(false, false, buildTableProperties(fileFormat))); + this.fileFormat = fileFormat; + System.setProperty(DeleteCache.DELETE_CACHE_ENABLED, "false"); CombinedDeleteFilter.FILTER_EQ_DELETE_TRIGGER_RECORD_COUNT = BLOOM_TRIGGER; } @@ -153,8 +150,11 @@ private GenericCombinedIcebergDataReader buildReader(RewriteFilesInput input) { * Verifies that with the bloom-filter path active, records matched by equality-delete files are * filtered out and the remainder survives — exercising the hoisted StructProjection code. */ - @Test - public void testBloomFilterWithHoistedProjection_singleDeleteSchema() throws IOException { + @ParameterizedTest(name = "fileFormat = {0}") + @MethodSource("parameters") + public void testBloomFilterWithHoistedProjection_singleDeleteSchema(FileFormat fileFormat) + throws IOException { + prepare(fileFormat); // 3 data rows: id=1,2,3 DataFile dataFile = writeDataFile( @@ -181,17 +181,18 @@ public void testBloomFilterWithHoistedProjection_singleDeleteSchema() throws IOE getMixedTable()); GenericCombinedIcebergDataReader reader = buildReader(input); - Assert.assertTrue("Bloom filter should be active", reader.getDeleteFilter().isFilterEqDelete()); + Assertions.assertTrue( + reader.getDeleteFilter().isFilterEqDelete(), "Bloom filter should be active"); try (CloseableIterable surviving = reader.readData()) { List result = Lists.newArrayList(surviving); - Assert.assertEquals("Only id=3 should survive", 1, result.size()); - Assert.assertEquals(3, result.get(0).get(0)); + Assertions.assertEquals(1, result.size(), "Only id=3 should survive"); + Assertions.assertEquals(3, result.get(0).get(0)); } try (CloseableIterable deleted = reader.readDeletedData()) { - Assert.assertEquals( - "id=1 and id=2 should be reported as deleted", 2, Iterables.size(deleted)); + Assertions.assertEquals( + 2, Iterables.size(deleted), "id=1 and id=2 should be reported as deleted"); } reader.close(); @@ -206,8 +207,11 @@ public void testBloomFilterWithHoistedProjection_singleDeleteSchema() throws IOE * Uses two equality-delete files whose schemas differ (id-only vs id+name). Both must be put into * the bloom filter correctly so that applyEqDeletesForSchema can later verify membership. */ - @Test - public void testBloomFilterWithHoistedProjection_multipleDeleteSchemas() throws IOException { + @ParameterizedTest(name = "fileFormat = {0}") + @MethodSource("parameters") + public void testBloomFilterWithHoistedProjection_multipleDeleteSchemas(FileFormat fileFormat) + throws IOException { + prepare(fileFormat); DataFile dataFile = writeDataFile( Arrays.asList( @@ -245,15 +249,16 @@ public void testBloomFilterWithHoistedProjection_multipleDeleteSchemas() throws getMixedTable()); GenericCombinedIcebergDataReader reader = buildReader(input); - Assert.assertTrue("Bloom filter should be active", reader.getDeleteFilter().isFilterEqDelete()); + Assertions.assertTrue( + reader.getDeleteFilter().isFilterEqDelete(), "Bloom filter should be active"); // id=1,2,3 are all deleted by eqDeleteById; none should survive try (CloseableIterable surviving = reader.readData()) { - Assert.assertEquals("All records should be deleted", 0, Iterables.size(surviving)); + Assertions.assertEquals(0, Iterables.size(surviving), "All records should be deleted"); } try (CloseableIterable deleted = reader.readDeletedData()) { - Assert.assertEquals("All 3 rows should appear as deleted", 3, Iterables.size(deleted)); + Assertions.assertEquals(3, Iterables.size(deleted), "All 3 rows should appear as deleted"); } reader.close(); @@ -268,8 +273,11 @@ public void testBloomFilterWithHoistedProjection_multipleDeleteSchemas() throws * Ensures false-negative freedom: records NOT covered by any equality-delete survive even when * the bloom filter path is active (i.e. the hoisted StructProjection wraps records faithfully). */ - @Test - public void testBloomFilterWithHoistedProjection_noFalseNegatives() throws IOException { + @ParameterizedTest(name = "fileFormat = {0}") + @MethodSource("parameters") + public void testBloomFilterWithHoistedProjection_noFalseNegatives(FileFormat fileFormat) + throws IOException { + prepare(fileFormat); DataFile dataFile = writeDataFile( Arrays.asList( @@ -296,11 +304,12 @@ public void testBloomFilterWithHoistedProjection_noFalseNegatives() throws IOExc getMixedTable()); GenericCombinedIcebergDataReader reader = buildReader(input); - Assert.assertTrue("Bloom filter should be active", reader.getDeleteFilter().isFilterEqDelete()); + Assertions.assertTrue( + reader.getDeleteFilter().isFilterEqDelete(), "Bloom filter should be active"); try (CloseableIterable surviving = reader.readData()) { List result = Lists.newArrayList(surviving); - Assert.assertEquals("id=20 and id=30 should survive", 2, result.size()); + Assertions.assertEquals(2, result.size(), "id=20 and id=30 should survive"); } reader.close(); @@ -315,8 +324,10 @@ public void testBloomFilterWithHoistedProjection_noFalseNegatives() throws IOExc * Resets the threshold above the delete-record count so the bloom filter is not activated. * Confirms the non-bloom code path still correctly applies equality deletes. */ - @Test - public void testEqualityDeleteWithoutBloomFilter() throws IOException { + @ParameterizedTest(name = "fileFormat = {0}") + @MethodSource("parameters") + public void testEqualityDeleteWithoutBloomFilter(FileFormat fileFormat) throws IOException { + prepare(fileFormat); // Set threshold high so bloom filter is NOT activated CombinedDeleteFilter.FILTER_EQ_DELETE_TRIGGER_RECORD_COUNT = 1_000_000L; @@ -341,12 +352,12 @@ public void testEqualityDeleteWithoutBloomFilter() throws IOException { getMixedTable()); GenericCombinedIcebergDataReader reader = buildReader(input); - Assert.assertFalse( - "Bloom filter should NOT be active", reader.getDeleteFilter().isFilterEqDelete()); + Assertions.assertFalse( + reader.getDeleteFilter().isFilterEqDelete(), "Bloom filter should NOT be active"); try (CloseableIterable surviving = reader.readData()) { List result = Lists.newArrayList(surviving); - Assert.assertEquals("id=1 and id=3 should survive", 2, result.size()); + Assertions.assertEquals(2, result.size(), "id=1 and id=3 should survive"); } reader.close(); diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestFileNameGenerator.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestFileNameGenerator.java index 015fbef1a4..c3b1fe5c50 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestFileNameGenerator.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestFileNameGenerator.java @@ -24,8 +24,8 @@ import org.apache.amoro.data.FileNameRules; import org.apache.amoro.io.writer.TaskWriterKey; import org.apache.iceberg.FileFormat; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; public class TestFileNameGenerator { @@ -37,17 +37,17 @@ public void newBaseFileName() { String fileName = fileNameGenerator.fileName(writerKey); System.out.println(fileName); DefaultKeyedFile.FileMeta fileMeta = FileNameRules.parseBase(fileName); - Assert.assertEquals(fileMeta.node(), DataTreeNode.of(3, 3)); - Assert.assertEquals(fileMeta.type(), DataFileType.BASE_FILE); - Assert.assertEquals(fileMeta.transactionId(), 2L); + Assertions.assertEquals(fileMeta.node(), DataTreeNode.of(3, 3)); + Assertions.assertEquals(fileMeta.type(), DataFileType.BASE_FILE); + Assertions.assertEquals(fileMeta.transactionId(), 2L); } @Test public void hiveFileName() { DefaultKeyedFile.FileMeta fileMeta = FileNameRules.parseBase("a"); - Assert.assertEquals(fileMeta.node(), DataTreeNode.of(0, 0)); - Assert.assertEquals(fileMeta.type(), DataFileType.BASE_FILE); - Assert.assertEquals(fileMeta.transactionId(), 0L); + Assertions.assertEquals(fileMeta.node(), DataTreeNode.of(0, 0)); + Assertions.assertEquals(fileMeta.type(), DataFileType.BASE_FILE); + Assertions.assertEquals(fileMeta.transactionId(), 0L); } @Test @@ -57,9 +57,9 @@ public void flinkChangeFile2Base() { new TaskWriterKey(null, DataTreeNode.of(3, 3), DataFileType.INSERT_FILE); String fileName = fileNameGenerator.fileName(writerKey); DefaultKeyedFile.FileMeta fileMeta = FileNameRules.parseBase(fileName); - Assert.assertEquals(fileMeta.node(), DataTreeNode.of(3, 3)); - Assert.assertEquals(fileMeta.type(), DataFileType.BASE_FILE); - Assert.assertEquals(fileMeta.transactionId(), 0L); + Assertions.assertEquals(fileMeta.node(), DataTreeNode.of(3, 3)); + Assertions.assertEquals(fileMeta.type(), DataFileType.BASE_FILE); + Assertions.assertEquals(fileMeta.transactionId(), 0L); } @Test @@ -69,9 +69,9 @@ public void flinkChangeFile() { new TaskWriterKey(null, DataTreeNode.of(3, 3), DataFileType.INSERT_FILE); String fileName = fileNameGenerator.fileName(writerKey); DefaultKeyedFile.FileMeta fileMeta = FileNameRules.parseChange(fileName, 5L); - Assert.assertEquals(fileMeta.node(), DataTreeNode.of(3, 3)); - Assert.assertEquals(fileMeta.type(), DataFileType.INSERT_FILE); - Assert.assertEquals(fileMeta.transactionId(), 5L); + Assertions.assertEquals(fileMeta.node(), DataTreeNode.of(3, 3)); + Assertions.assertEquals(fileMeta.type(), DataFileType.INSERT_FILE); + Assertions.assertEquals(fileMeta.transactionId(), 5L); } @Test @@ -81,9 +81,9 @@ public void sparkChangeFile() { new TaskWriterKey(null, DataTreeNode.of(3, 3), DataFileType.INSERT_FILE); String fileName = fileNameGenerator.fileName(writerKey); DefaultKeyedFile.FileMeta fileMeta = FileNameRules.parseChange(fileName, 6L); - Assert.assertEquals(fileMeta.node(), DataTreeNode.of(3, 3)); - Assert.assertEquals(fileMeta.type(), DataFileType.INSERT_FILE); - Assert.assertEquals(fileMeta.transactionId(), 5L); + Assertions.assertEquals(fileMeta.node(), DataTreeNode.of(3, 3)); + Assertions.assertEquals(fileMeta.type(), DataFileType.INSERT_FILE); + Assertions.assertEquals(fileMeta.transactionId(), 5L); } @Test @@ -92,27 +92,27 @@ public void adaptOldFileName() { FileNameRules.parseChange( "hdfs://easyops-sloth/user/warehouse/animal_partition_two/base/5-I-2-00000-941953957-0000000001.parquet", 6L); - Assert.assertEquals(fileMeta.node(), DataTreeNode.of(3, 1)); - Assert.assertEquals(fileMeta.type(), DataFileType.INSERT_FILE); - Assert.assertEquals(fileMeta.transactionId(), 2L); + Assertions.assertEquals(fileMeta.node(), DataTreeNode.of(3, 1)); + Assertions.assertEquals(fileMeta.type(), DataFileType.INSERT_FILE); + Assertions.assertEquals(fileMeta.transactionId(), 2L); } @Test public void testHiddenFileParse() { String path = "hdfs://test/animal_partition_two/hive/.5-I-2-00000-941953957-0000000001.parquet"; DefaultKeyedFile.FileMeta fileMeta = FileNameRules.parseBase(path); - Assert.assertEquals(fileMeta.node(), DataTreeNode.of(3, 1)); - Assert.assertEquals(fileMeta.type(), DataFileType.BASE_FILE); - Assert.assertEquals(fileMeta.transactionId(), 2L); + Assertions.assertEquals(fileMeta.node(), DataTreeNode.of(3, 1)); + Assertions.assertEquals(fileMeta.type(), DataFileType.BASE_FILE); + Assertions.assertEquals(fileMeta.transactionId(), 2L); long txId = FileNameRules.parseTransactionId(path); - Assert.assertEquals(2L, txId); + Assertions.assertEquals(2L, txId); String invalidPath = "hdfs://test/animal_partition_two/hive/A5-I-2-00000-941953957-0000000001.parquet"; fileMeta = FileNameRules.parseBase(invalidPath); - Assert.assertEquals(fileMeta.node(), DataTreeNode.of(0, 0)); - Assert.assertEquals(fileMeta.type(), DataFileType.BASE_FILE); - Assert.assertEquals(fileMeta.transactionId(), 0L); + Assertions.assertEquals(fileMeta.node(), DataTreeNode.of(0, 0)); + Assertions.assertEquals(fileMeta.type(), DataFileType.BASE_FILE); + Assertions.assertEquals(fileMeta.transactionId(), 0L); } } diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestIcebergCombinedReader.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestIcebergCombinedReader.java index 442efaf74b..dc5d391b0d 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestIcebergCombinedReader.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestIcebergCombinedReader.java @@ -45,11 +45,10 @@ import org.apache.iceberg.io.OutputFileFactory; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.util.Pair; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; import java.util.ArrayList; @@ -58,11 +57,11 @@ import java.util.List; import java.util.Map; import java.util.stream.IntStream; +import java.util.stream.Stream; -@RunWith(Parameterized.class) public class TestIcebergCombinedReader extends TableTestBase { - private final FileFormat fileFormat; + private FileFormat fileFormat; private RewriteFilesInput scanTask; @@ -70,36 +69,20 @@ public class TestIcebergCombinedReader extends TableTestBase { private RewriteFilesInput filterEqDeleteScanTask; - public TestIcebergCombinedReader( - boolean partitionedTable, FileFormat fileFormat, boolean deleteCacheEnabled) { - super( - new BasicCatalogTestHelper(TableFormat.ICEBERG), - new BasicTableTestHelper(false, partitionedTable, buildTableProperties(fileFormat))); - this.fileFormat = fileFormat; - if (deleteCacheEnabled) { - System.setProperty(DeleteCache.DELETE_CACHE_ENABLED, "true"); - } else { - System.setProperty(DeleteCache.DELETE_CACHE_ENABLED, "false"); - } - } - - @Parameterized.Parameters( - name = "partitionedTable = {0}, fileFormat = {1}, deleteCacheEnabled = {2}") - public static Object[][] parameters() { - return new Object[][] { - {true, FileFormat.PARQUET, true}, - {false, FileFormat.PARQUET, true}, - {true, FileFormat.AVRO, true}, - {false, FileFormat.AVRO, true}, - {true, FileFormat.ORC, true}, - {false, FileFormat.ORC, true}, - {true, FileFormat.PARQUET, false}, - {false, FileFormat.PARQUET, false}, - {true, FileFormat.AVRO, false}, - {false, FileFormat.AVRO, false}, - {true, FileFormat.ORC, false}, - {false, FileFormat.ORC, false} - }; + public static Stream parameters() { + return Stream.of( + Arguments.of(true, FileFormat.PARQUET, true), + Arguments.of(false, FileFormat.PARQUET, true), + Arguments.of(true, FileFormat.AVRO, true), + Arguments.of(false, FileFormat.AVRO, true), + Arguments.of(true, FileFormat.ORC, true), + Arguments.of(false, FileFormat.ORC, true), + Arguments.of(true, FileFormat.PARQUET, false), + Arguments.of(false, FileFormat.PARQUET, false), + Arguments.of(true, FileFormat.AVRO, false), + Arguments.of(false, FileFormat.AVRO, false), + Arguments.of(true, FileFormat.ORC, false), + Arguments.of(false, FileFormat.ORC, false)); } private static Map buildTableProperties(FileFormat fileFormat) { @@ -110,6 +93,20 @@ private static Map buildTableProperties(FileFormat fileFormat) { return tableProperties; } + private void prepare(boolean partitionedTable, FileFormat fileFormat, boolean deleteCacheEnabled) + throws IOException { + setupTable( + new BasicCatalogTestHelper(TableFormat.ICEBERG), + new BasicTableTestHelper(false, partitionedTable, buildTableProperties(fileFormat))); + this.fileFormat = fileFormat; + if (deleteCacheEnabled) { + System.setProperty(DeleteCache.DELETE_CACHE_ENABLED, "true"); + } else { + System.setProperty(DeleteCache.DELETE_CACHE_ENABLED, "false"); + } + initDataAndReader(); + } + private StructLike getPartitionData() { if (isPartitionedTable()) { return TestHelpers.Row.of(0); @@ -118,8 +115,7 @@ private StructLike getPartitionData() { } } - @Before - public void initDataAndReader() throws IOException { + private void initDataAndReader() throws IOException { StructLike partitionData = getPartitionData(); OutputFileFactory outputFileFactory = OutputFileFactory.builderFor(getMixedTable().asUnkeyedTable(), 0, 1) @@ -201,8 +197,12 @@ public void initDataAndReader() throws IOException { getMixedTable()); } - @Test - public void readAllData() throws IOException { + @ParameterizedTest(name = "partitionedTable = {0}, fileFormat = {1}, deleteCacheEnabled = {2}") + @MethodSource("parameters") + public void readAllData( + boolean partitionedTable, FileFormat fileFormat, boolean deleteCacheEnabled) + throws IOException { + prepare(partitionedTable, fileFormat, deleteCacheEnabled); GenericCombinedIcebergDataReader dataReader = new GenericCombinedIcebergDataReader( getMixedTable().io(), @@ -217,15 +217,19 @@ public void readAllData() throws IOException { scanTask, ""); try (CloseableIterable records = dataReader.readData()) { - Assert.assertEquals(1, Iterables.size(records)); + Assertions.assertEquals(1, Iterables.size(records)); Record record = Iterables.getFirst(records, null); - Assert.assertEquals(record.get(0), 3); + Assertions.assertEquals(record.get(0), 3); } dataReader.close(); } - @Test - public void readAllDataNegate() throws IOException { + @ParameterizedTest(name = "partitionedTable = {0}, fileFormat = {1}, deleteCacheEnabled = {2}") + @MethodSource("parameters") + public void readAllDataNegate( + boolean partitionedTable, FileFormat fileFormat, boolean deleteCacheEnabled) + throws IOException { + prepare(partitionedTable, fileFormat, deleteCacheEnabled); GenericCombinedIcebergDataReader dataReader = new GenericCombinedIcebergDataReader( getMixedTable().io(), @@ -240,17 +244,21 @@ public void readAllDataNegate() throws IOException { scanTask, ""); try (CloseableIterable records = dataReader.readDeletedData()) { - Assert.assertEquals(2, Iterables.size(records)); + Assertions.assertEquals(2, Iterables.size(records)); Record first = Iterables.getFirst(records, null); - Assert.assertEquals(first.get(1), 0L); + Assertions.assertEquals(first.get(1), 0L); Record last = Iterables.getLast(records); - Assert.assertEquals(last.get(1), 1L); + Assertions.assertEquals(last.get(1), 1L); } dataReader.close(); } - @Test - public void readOnlyData() throws IOException { + @ParameterizedTest(name = "partitionedTable = {0}, fileFormat = {1}, deleteCacheEnabled = {2}") + @MethodSource("parameters") + public void readOnlyData( + boolean partitionedTable, FileFormat fileFormat, boolean deleteCacheEnabled) + throws IOException { + prepare(partitionedTable, fileFormat, deleteCacheEnabled); GenericCombinedIcebergDataReader dataReader = new GenericCombinedIcebergDataReader( getMixedTable().io(), @@ -265,13 +273,17 @@ public void readOnlyData() throws IOException { dataScanTask, ""); try (CloseableIterable records = dataReader.readData()) { - Assert.assertEquals(3, Iterables.size(records)); + Assertions.assertEquals(3, Iterables.size(records)); } dataReader.close(); } - @Test - public void readOnlyDataNegate() throws IOException { + @ParameterizedTest(name = "partitionedTable = {0}, fileFormat = {1}, deleteCacheEnabled = {2}") + @MethodSource("parameters") + public void readOnlyDataNegate( + boolean partitionedTable, FileFormat fileFormat, boolean deleteCacheEnabled) + throws IOException { + prepare(partitionedTable, fileFormat, deleteCacheEnabled); GenericCombinedIcebergDataReader dataReader = new GenericCombinedIcebergDataReader( getMixedTable().io(), @@ -286,13 +298,17 @@ public void readOnlyDataNegate() throws IOException { dataScanTask, ""); try (CloseableIterable records = dataReader.readDeletedData()) { - Assert.assertEquals(0, Iterables.size(records)); + Assertions.assertEquals(0, Iterables.size(records)); } dataReader.close(); } - @Test - public void readDataEnableFilterEqDelete() throws IOException { + @ParameterizedTest(name = "partitionedTable = {0}, fileFormat = {1}, deleteCacheEnabled = {2}") + @MethodSource("parameters") + public void readDataEnableFilterEqDelete( + boolean partitionedTable, FileFormat fileFormat, boolean deleteCacheEnabled) + throws IOException { + prepare(partitionedTable, fileFormat, deleteCacheEnabled); CombinedDeleteFilter.FILTER_EQ_DELETE_TRIGGER_RECORD_COUNT = 100L; GenericCombinedIcebergDataReader dataReader = new GenericCombinedIcebergDataReader( @@ -308,20 +324,24 @@ public void readDataEnableFilterEqDelete() throws IOException { filterEqDeleteScanTask, ""); - Assert.assertTrue(dataReader.getDeleteFilter().isFilterEqDelete()); + Assertions.assertTrue(dataReader.getDeleteFilter().isFilterEqDelete()); try (CloseableIterable records = dataReader.readData()) { - Assert.assertEquals(1, Iterables.size(records)); + Assertions.assertEquals(1, Iterables.size(records)); } try (CloseableIterable records = dataReader.readDeletedData()) { - Assert.assertEquals(2, Iterables.size(records)); + Assertions.assertEquals(2, Iterables.size(records)); } dataReader.close(); } - @Test - public void readDataDropAEqField() throws IOException { + @ParameterizedTest(name = "partitionedTable = {0}, fileFormat = {1}, deleteCacheEnabled = {2}") + @MethodSource("parameters") + public void readDataDropAEqField( + boolean partitionedTable, FileFormat fileFormat, boolean deleteCacheEnabled) + throws IOException { + prepare(partitionedTable, fileFormat, deleteCacheEnabled); CombinedDeleteFilter.FILTER_EQ_DELETE_TRIGGER_RECORD_COUNT = 100L; StructLike partitionData = getPartitionData(); OutputFileFactory outputFileFactory = @@ -388,18 +408,22 @@ public void readDataDropAEqField() throws IOException { task2, ""); try (CloseableIterable readRecords = dataReader.readData()) { - Assert.assertEquals(1, Iterables.size(readRecords)); + Assertions.assertEquals(1, Iterables.size(readRecords)); } try (CloseableIterable readRecords = dataReader.readDeletedData()) { - Assert.assertEquals(2, Iterables.size(readRecords)); + Assertions.assertEquals(2, Iterables.size(readRecords)); } dataReader.close(); } - @Test - public void readDataReplaceAEqField() throws IOException { + @ParameterizedTest(name = "partitionedTable = {0}, fileFormat = {1}, deleteCacheEnabled = {2}") + @MethodSource("parameters") + public void readDataReplaceAEqField( + boolean partitionedTable, FileFormat fileFormat, boolean deleteCacheEnabled) + throws IOException { + prepare(partitionedTable, fileFormat, deleteCacheEnabled); StructLike partitionData = getPartitionData(); OutputFileFactory outputFileFactory = OutputFileFactory.builderFor(getMixedTable().asUnkeyedTable(), 0, 1) @@ -463,17 +487,21 @@ public void readDataReplaceAEqField() throws IOException { task, ""); try (CloseableIterable readRecords = dataReader.readData()) { - Assert.assertEquals(0, Iterables.size(readRecords)); + Assertions.assertEquals(0, Iterables.size(readRecords)); } try (CloseableIterable readRecords = dataReader.readDeletedData()) { - Assert.assertEquals(3, Iterables.size(readRecords)); + Assertions.assertEquals(3, Iterables.size(readRecords)); } dataReader.close(); } - @Test - public void readReadAddAEqField() throws IOException { + @ParameterizedTest(name = "partitionedTable = {0}, fileFormat = {1}, deleteCacheEnabled = {2}") + @MethodSource("parameters") + public void readReadAddAEqField( + boolean partitionedTable, FileFormat fileFormat, boolean deleteCacheEnabled) + throws IOException { + prepare(partitionedTable, fileFormat, deleteCacheEnabled); StructLike partitionData = getPartitionData(); OutputFileFactory outputFileFactory = OutputFileFactory.builderFor(getMixedTable().asUnkeyedTable(), 0, 1) @@ -539,10 +567,10 @@ public void readReadAddAEqField() throws IOException { ""); try (CloseableIterable readRecords = dataReader.readData()) { - Assert.assertEquals(0, Iterables.size(readRecords)); + Assertions.assertEquals(0, Iterables.size(readRecords)); } try (CloseableIterable readRecords = dataReader.readDeletedData()) { - Assert.assertEquals(3, Iterables.size(readRecords)); + Assertions.assertEquals(3, Iterables.size(readRecords)); } dataReader.close(); diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestIcebergCombinedReaderVariousTypes.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestIcebergCombinedReaderVariousTypes.java index 40af54ae34..830c814a59 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestIcebergCombinedReaderVariousTypes.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestIcebergCombinedReaderVariousTypes.java @@ -49,22 +49,31 @@ import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.jetbrains.annotations.NotNull; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.stream.Stream; -@RunWith(Parameterized.class) public class TestIcebergCombinedReaderVariousTypes extends TableTestBase { - public TestIcebergCombinedReaderVariousTypes(Schema schema) { - super( + public static Stream parameters() { + return Stream.of( + Arguments.of(getSchema(Types.DateType.get())), + Arguments.of(getSchema(Types.TimeType.get())), + Arguments.of(getSchema(Types.TimestampType.withoutZone())), + Arguments.of(getSchema(Types.TimestampType.withZone())), + Arguments.of(getSchema(Types.DecimalType.of(5, 2)))); + } + + private void prepare(Schema schema) throws IOException { + setupTable( new BasicCatalogTestHelper(TableFormat.ICEBERG), new BasicTableTestHelper( schema, @@ -73,23 +82,6 @@ public TestIcebergCombinedReaderVariousTypes(Schema schema) { buildTableProperties())); } - @Parameterized.Parameters(name = "schema = {0}") - public static Object[] parameters() { - Schema dateSchema = getSchema(Types.DateType.get()); - - Schema timeSchema = getSchema(Types.TimeType.get()); - - Schema timestampWithoutZoneSchema = getSchema(Types.TimestampType.withoutZone()); - - Schema timestampWithZoneSchema = getSchema(Types.TimestampType.withZone()); - - Schema decimalSchema = getSchema(Types.DecimalType.of(5, 2)); - - return new Object[] { - dateSchema, timeSchema, timestampWithoutZoneSchema, timestampWithZoneSchema, decimalSchema - }; - } - @NotNull private static Schema getSchema(Type type) { return new Schema( @@ -108,8 +100,10 @@ private static Map buildTableProperties() { return tableProperties; } - @Test - public void valid() throws IOException { + @ParameterizedTest(name = "schema = {0}") + @MethodSource("parameters") + public void valid(Schema schema) throws IOException { + prepare(schema); UnkeyedTable table = getMixedTable().asUnkeyedTable(); Record record = RandomGenericData.generate(table.schema(), 1, 1).get(0); @@ -150,11 +144,13 @@ public void valid() throws IOException { "") .readData(); - Assert.assertEquals(Iterables.size(readData), 1); + Assertions.assertEquals(Iterables.size(readData), 1); } - @Test - public void readDataEnableFilterEqDelete() throws IOException { + @ParameterizedTest(name = "schema = {0}") + @MethodSource("parameters") + public void readDataEnableFilterEqDelete(Schema schema) throws IOException { + prepare(schema); UnkeyedTable table = getMixedTable().asUnkeyedTable(); List records = RandomGenericData.generate(table.schema(), 50, 1); List deleteRecords = RandomGenericData.generate(table.schema(), 200, 1); @@ -179,8 +175,8 @@ public void readDataEnableFilterEqDelete() throws IOException { DataFile[] dataFiles = dataFileList.toArray(new DataFile[0]); DeleteFile[] deleteFiles = deleteFileList.toArray(new DeleteFile[0]); - Assert.assertNotEquals(dataFiles.length, 0); - Assert.assertNotEquals(deleteFiles.length, 0); + Assertions.assertNotEquals(dataFiles.length, 0); + Assertions.assertNotEquals(deleteFiles.length, 0); RewriteFilesInput input = new RewriteFilesInput( @@ -200,10 +196,10 @@ public void readDataEnableFilterEqDelete() throws IOException { null, input, ""); - Assert.assertTrue(reader.getDeleteFilter().isFilterEqDelete()); + Assertions.assertTrue(reader.getDeleteFilter().isFilterEqDelete()); CloseableIterable readData = reader.readData(); - Assert.assertEquals(Iterables.size(readData), 0); + Assertions.assertEquals(Iterables.size(readData), 0); } private static void write(UnkeyedTable table, List list) throws IOException { diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestIcebergFanoutPosDeleteWriter.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestIcebergFanoutPosDeleteWriter.java index b8667ce227..6af2e44b80 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestIcebergFanoutPosDeleteWriter.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestIcebergFanoutPosDeleteWriter.java @@ -37,37 +37,38 @@ import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; import org.apache.iceberg.io.DeleteSchemaUtil; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import java.util.stream.Stream; -@RunWith(Parameterized.class) public class TestIcebergFanoutPosDeleteWriter extends TableTestBase { - private final FileFormat fileFormat; + private FileFormat fileFormat; - public TestIcebergFanoutPosDeleteWriter(boolean partitionedTable, FileFormat fileFormat) { - super( + public static Stream parameters() { + return Stream.of( + Arguments.of(true, FileFormat.PARQUET), + Arguments.of(false, FileFormat.PARQUET), + Arguments.of(true, FileFormat.AVRO), + Arguments.of(false, FileFormat.AVRO), + Arguments.of(true, FileFormat.ORC), + Arguments.of(false, FileFormat.ORC)); + } + + private void prepare(boolean partitionedTable, FileFormat fileFormat) throws IOException { + setupTable( new BasicCatalogTestHelper(TableFormat.ICEBERG), new BasicTableTestHelper(false, partitionedTable)); this.fileFormat = fileFormat; } - @Parameterized.Parameters(name = "partitionedTable = {0}, fileFormat = {1}") - public static Object[][] parameters() { - return new Object[][] { - {true, FileFormat.PARQUET}, {false, FileFormat.PARQUET}, - {true, FileFormat.AVRO}, {false, FileFormat.AVRO}, - {true, FileFormat.ORC}, {false, FileFormat.ORC} - }; - } - private StructLike getPartitionData() { if (isPartitionedTable()) { return TestHelpers.Row.of(0); @@ -76,8 +77,11 @@ private StructLike getPartitionData() { } } - @Test - public void testWritePosDelete() throws IOException { + @ParameterizedTest(name = "partitionedTable = {0}, fileFormat = {1}") + @MethodSource("parameters") + public void testWritePosDelete(boolean partitionedTable, FileFormat fileFormat) + throws IOException { + prepare(partitionedTable, fileFormat); StructLike partitionData = getPartitionData(); GenericAppenderFactory appenderFactory = new GenericAppenderFactory(getMixedTable().schema(), getMixedTable().spec()); @@ -117,7 +121,7 @@ public void testWritePosDelete() throws IOException { icebergPosDeleteWriter.delete(dataFile2Path, 8); List deleteFiles = icebergPosDeleteWriter.complete(); - Assert.assertEquals(2, deleteFiles.size()); + Assertions.assertEquals(2, deleteFiles.size()); Map deleteFileMap = deleteFiles.stream().collect(Collectors.toMap(f -> f.path().toString(), f -> f)); DeleteFile deleteFile1 = @@ -126,10 +130,10 @@ public void testWritePosDelete() throws IOException { TableFileUtil.getNewFilePath( dataDir, fileFormat.addExtension("data-1-delete-suffix"))) .toString()); - Assert.assertNotNull(deleteFile1); - Assert.assertTrue( + Assertions.assertNotNull(deleteFile1); + Assertions.assertTrue( TableFileUtil.isOptimizingPosDeleteFile(dataFile1Path, deleteFile1.path().toString())); - Assert.assertEquals(3, deleteFile1.recordCount()); + Assertions.assertEquals(3, deleteFile1.recordCount()); // Check whether the path-pos pairs are sorted as expected. Schema pathPosSchema = DeleteSchemaUtil.pathPosSchema(); Record record = GenericRecord.create(pathPosSchema); @@ -138,7 +142,7 @@ public void testWritePosDelete() throws IOException { record.copy("file_path", dataFile1Path, "pos", 0L), record.copy("file_path", dataFile1Path, "pos", 1L), record.copy("file_path", dataFile1Path, "pos", 3L)); - Assert.assertEquals( + Assertions.assertEquals( expectedDeletes, MixedDataTestHelpers.readDataFile(fileFormat, pathPosSchema, deleteFile1.path())); @@ -148,23 +152,23 @@ public void testWritePosDelete() throws IOException { TableFileUtil.getNewFilePath( dataDir, fileFormat.addExtension("data-2-delete-suffix"))) .toString()); - Assert.assertNotNull(deleteFile2); - Assert.assertTrue( + Assertions.assertNotNull(deleteFile2); + Assertions.assertTrue( TableFileUtil.isOptimizingPosDeleteFile(dataFile2Path, deleteFile2.path().toString())); - Assert.assertEquals( + Assertions.assertEquals( new Path( TableFileUtil.getNewFilePath( dataDir, fileFormat.addExtension("data-2-delete-suffix"))) .toString(), deleteFile2.path().toString()); - Assert.assertEquals(3, deleteFile2.recordCount()); + Assertions.assertEquals(3, deleteFile2.recordCount()); // Check whether the path-pos pairs are sorted as expected. expectedDeletes = Lists.newArrayList( record.copy("file_path", dataFile2Path, "pos", 8L), record.copy("file_path", dataFile2Path, "pos", 9L), record.copy("file_path", dataFile2Path, "pos", 10L)); - Assert.assertEquals( + Assertions.assertEquals( expectedDeletes, MixedDataTestHelpers.readDataFile(fileFormat, pathPosSchema, deleteFile2.path())); } diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestMixedTreeNodePosDeleteWriter.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestMixedTreeNodePosDeleteWriter.java index 2f58540ac0..30f6e7c5f5 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestMixedTreeNodePosDeleteWriter.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestMixedTreeNodePosDeleteWriter.java @@ -33,32 +33,32 @@ import org.apache.iceberg.data.GenericAppenderFactory; import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; import java.util.List; +import java.util.stream.Stream; -@RunWith(Parameterized.class) public class TestMixedTreeNodePosDeleteWriter extends TableTestBase { - public TestMixedTreeNodePosDeleteWriter( - CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) { - super(catalogTestHelper, tableTestHelper); + public static Stream parameters() { + return Stream.of( + Arguments.of( + new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), + new BasicTableTestHelper(true, true)), + Arguments.of( + new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), + new BasicTableTestHelper(true, false))); } - @Parameterized.Parameters(name = "{1},{2}") - public static Object[] parameters() { - return new Object[][] { - {new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), new BasicTableTestHelper(true, true)}, - {new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), new BasicTableTestHelper(true, false)} - }; - } - - @Test - public void test() throws IOException { + @ParameterizedTest(name = "{0},{1}") + @MethodSource("parameters") + public void test(CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) + throws IOException { + setupTable(catalogTestHelper, tableTestHelper); UnkeyedTable table = getMixedTable().asKeyedTable().baseTable(); GenericAppenderFactory appenderFactory = new GenericAppenderFactory(table.schema(), table.spec()); @@ -91,6 +91,6 @@ public void test() throws IOException { List complete = writer.complete(); - Assert.assertEquals(complete.size(), 4); + Assertions.assertEquals(complete.size(), 4); } } diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestRecoverableAuthenticatedFileIO.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestRecoverableAuthenticatedFileIO.java index e05269edc4..32682568ed 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestRecoverableAuthenticatedFileIO.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestRecoverableAuthenticatedFileIO.java @@ -26,9 +26,9 @@ import org.apache.amoro.table.MixedTable; import org.apache.amoro.table.TableProperties; import org.apache.iceberg.io.OutputFile; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.io.IOException; @@ -40,14 +40,11 @@ public class TestRecoverableAuthenticatedFileIO extends TableTestBase { private String file2; private String file3; - public TestRecoverableAuthenticatedFileIO() { - super( + @BeforeEach + public void before() throws IOException { + setupTable( new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), new BasicTableTestHelper(true, true)); - } - - @Before - public void before() { MixedTable mixedTable = getMixedTable(); trashManager = TableTrashManagers.build( @@ -68,8 +65,8 @@ public void before() { @Test public void exists() throws IOException { createFile(file1); - Assert.assertTrue(recoverableHadoopFileIO.exists(file1)); - Assert.assertFalse(recoverableHadoopFileIO.exists(file2)); + Assertions.assertTrue(recoverableHadoopFileIO.exists(file1)); + Assertions.assertFalse(recoverableHadoopFileIO.exists(file2)); } @Test @@ -77,8 +74,8 @@ public void rename() throws IOException { String newLocation = getMixedTable().location() + "/base/test/test4.parquet"; createFile(file1); recoverableHadoopFileIO.rename(file1, newLocation); - Assert.assertFalse(authenticatedFileIO.exists(file1)); - Assert.assertTrue(authenticatedFileIO.exists(newLocation)); + Assertions.assertFalse(authenticatedFileIO.exists(file1)); + Assertions.assertTrue(authenticatedFileIO.exists(newLocation)); } @Test @@ -88,7 +85,7 @@ public void deleteDirectoryRecursively() throws IOException { createFile(file3); String dir = getMixedTable().location() + "/base/test"; recoverableHadoopFileIO.deletePrefix(dir); - Assert.assertFalse(authenticatedFileIO.exists(dir)); + Assertions.assertFalse(authenticatedFileIO.exists(dir)); } @Test @@ -98,68 +95,68 @@ public void list() throws IOException { createFile(file3); Iterable items = recoverableHadoopFileIO.listDirectory(getMixedTable().location() + "/base/test"); - Assert.assertEquals(3L, Streams.stream(items).count()); + Assertions.assertEquals(3L, Streams.stream(items).count()); } @Test public void isDirectory() throws IOException { createFile(file1); - Assert.assertFalse(recoverableHadoopFileIO.isDirectory(file1)); - Assert.assertTrue(recoverableHadoopFileIO.isDirectory(getMixedTable().location())); + Assertions.assertFalse(recoverableHadoopFileIO.isDirectory(file1)); + Assertions.assertTrue(recoverableHadoopFileIO.isDirectory(getMixedTable().location())); } @Test public void isEmptyDirectory() { String dir = getMixedTable().location() + "/location"; authenticatedFileIO.asFileSystemIO().makeDirectories(dir); - Assert.assertTrue(recoverableHadoopFileIO.isEmptyDirectory(dir)); - Assert.assertFalse(recoverableHadoopFileIO.isEmptyDirectory(getMixedTable().location())); + Assertions.assertTrue(recoverableHadoopFileIO.isEmptyDirectory(dir)); + Assertions.assertFalse(recoverableHadoopFileIO.isEmptyDirectory(getMixedTable().location())); } @Test public void deleteFile() throws IOException { createFile(file1); recoverableHadoopFileIO.deleteFile(file1); - Assert.assertFalse(authenticatedFileIO.exists(file1)); - Assert.assertTrue(trashManager.fileExistInTrash(file1)); + Assertions.assertFalse(authenticatedFileIO.exists(file1)); + Assertions.assertTrue(trashManager.fileExistInTrash(file1)); } @Test public void deleteInputFile() throws IOException { createFile(file1); recoverableHadoopFileIO.deleteFile(recoverableHadoopFileIO.newInputFile(file1)); - Assert.assertFalse(authenticatedFileIO.exists(file1)); - Assert.assertTrue(trashManager.fileExistInTrash(file1)); + Assertions.assertFalse(authenticatedFileIO.exists(file1)); + Assertions.assertTrue(trashManager.fileExistInTrash(file1)); } @Test public void deleteOutputFile() throws IOException { createFile(file1); recoverableHadoopFileIO.deleteFile(recoverableHadoopFileIO.newOutputFile(file1)); - Assert.assertFalse(authenticatedFileIO.exists(file1)); - Assert.assertTrue(trashManager.fileExistInTrash(file1)); + Assertions.assertFalse(authenticatedFileIO.exists(file1)); + Assertions.assertTrue(trashManager.fileExistInTrash(file1)); } @Test public void trashFilePattern() { - Assert.assertTrue(recoverableHadoopFileIO.matchTrashFilePattern(file1)); - Assert.assertTrue(recoverableHadoopFileIO.matchTrashFilePattern(file2)); - Assert.assertTrue(recoverableHadoopFileIO.matchTrashFilePattern(file3)); - Assert.assertTrue( + Assertions.assertTrue(recoverableHadoopFileIO.matchTrashFilePattern(file1)); + Assertions.assertTrue(recoverableHadoopFileIO.matchTrashFilePattern(file2)); + Assertions.assertTrue(recoverableHadoopFileIO.matchTrashFilePattern(file3)); + Assertions.assertTrue( recoverableHadoopFileIO.matchTrashFilePattern( getMixedTable().location() + "/metadata/version-hint.text")); - Assert.assertTrue( + Assertions.assertTrue( recoverableHadoopFileIO.matchTrashFilePattern( getMixedTable().location() + "/metadata/v2.metadata.json")); - Assert.assertTrue( + Assertions.assertTrue( recoverableHadoopFileIO.matchTrashFilePattern( getMixedTable().location() + "/metadata/snap-1515213806302741636-1-85fc817e-941d-4e9a-ab41-2dbf7687bfcd.avro")); - Assert.assertTrue( + Assertions.assertTrue( recoverableHadoopFileIO.matchTrashFilePattern( getMixedTable().location() + "/metadata/3ce7600d-4853-45d0-8533-84c12a611916-m0.avro")); - Assert.assertFalse( + Assertions.assertFalse( recoverableHadoopFileIO.matchTrashFilePattern( getMixedTable().location() + "/metadata/3ce7600d-4853-45d0-8533-84c12a611916.avro")); } diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/log/TestLogDataJsonSerialization.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/log/TestLogDataJsonSerialization.java index e59d2645dc..5efc31ad33 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/log/TestLogDataJsonSerialization.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/log/TestLogDataJsonSerialization.java @@ -18,13 +18,13 @@ package org.apache.amoro.log; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import org.apache.amoro.data.ChangeAction; import org.apache.amoro.utils.IdGenerator; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.math.BigDecimal; @@ -96,7 +96,7 @@ public void testLogDataSerialize() throws IOException { byte[] bytes = logDataJsonSerialization.serialize(logData); - Assert.assertNotNull(bytes); + assertNotNull(bytes); String actualJson = new String(Bytes.subByte(bytes, 18, bytes.length - 18)); String expected = "{\"f_boolean\":true,\"f_int\":1,\"f_long\":123456789,\"f_struct\":{\"f_sub_boolean\":false,\"f_sub_int\":2," @@ -112,7 +112,7 @@ public void testLogDataSerialize() throws IOException { LogDataJsonDeserialization logDataJsonDeserialization = new LogDataJsonDeserialization<>(userSchema, factory, arrayFactory, mapFactory); LogData result = logDataJsonDeserialization.deserialize(bytes); - Assert.assertNotNull(result); + assertNotNull(result); check(logData, result); } diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/op/TestOverwriteBaseFile.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/op/TestOverwriteBaseFile.java index 5a1daf750a..9e58a60b06 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/op/TestOverwriteBaseFile.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/op/TestOverwriteBaseFile.java @@ -27,14 +27,21 @@ import org.apache.iceberg.data.Record; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.util.StructLikeMap; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import java.io.IOException; import java.util.List; import java.util.Set; public class TestOverwriteBaseFile extends TableDataTestBase { + @BeforeEach + public void setUp() throws IOException { + initData(); + } + /** overwrite all partition, add new data files */ @Test public void testOverwriteAllPartition() { @@ -60,22 +67,22 @@ public void testOverwriteAllPartition() { StructLikeMap partitionOptimizedSequence = MixedTableUtil.readOptimizedSequence(getMixedTable().asKeyedTable()); // expect result: all partition with new txId - Assert.assertEquals( + Assertions.assertEquals( txId, partitionOptimizedSequence .get(MixedDataTestHelpers.recordPartition("2022-01-01T12:00:00")) .longValue()); - Assert.assertEquals( + Assertions.assertEquals( txId, partitionOptimizedSequence .get(MixedDataTestHelpers.recordPartition("2022-01-02T12:00:00")) .longValue()); - Assert.assertEquals( + Assertions.assertEquals( txId, partitionOptimizedSequence .get(MixedDataTestHelpers.recordPartition("2022-01-03T12:00:00")) .longValue()); - Assert.assertEquals( + Assertions.assertEquals( txId, partitionOptimizedSequence .get(MixedDataTestHelpers.recordPartition("2022-01-04T12:00:00")) @@ -105,18 +112,18 @@ public void testOverwriteAllPartition() { MixedDataTestHelpers.readKeyedTable( getMixedTable().asKeyedTable(), Expressions.alwaysTrue()); // partition1 -> base[7,8,9] - Assert.assertEquals(3, rows.size()); + Assertions.assertEquals(3, rows.size()); Set resultIdSet = Sets.newHashSet(); rows.forEach(r -> resultIdSet.add((Integer) r.get(0))); - Assert.assertTrue(resultIdSet.contains(7)); - Assert.assertTrue(resultIdSet.contains(8)); - Assert.assertTrue(resultIdSet.contains(9)); + Assertions.assertTrue(resultIdSet.contains(7)); + Assertions.assertTrue(resultIdSet.contains(8)); + Assertions.assertTrue(resultIdSet.contains(9)); } private void assertRange(long from, long to, long actual) { - Assert.assertTrue(actual >= from); - Assert.assertTrue(actual <= to); + Assertions.assertTrue(actual >= from); + Assertions.assertTrue(actual <= to); } @Test @@ -147,20 +154,20 @@ public void testOverwritePartitionByExpression() { StructLikeMap partitionOptimizedSequence = MixedTableUtil.readOptimizedSequence(getMixedTable().asKeyedTable()); // expect result: 1,2,4 partition with new txId, 3 partition is null - Assert.assertEquals( + Assertions.assertEquals( txId, partitionOptimizedSequence .get(MixedDataTestHelpers.recordPartition("2022-01-01T12:00:00")) .longValue()); - Assert.assertEquals( + Assertions.assertEquals( txId, partitionOptimizedSequence .get(MixedDataTestHelpers.recordPartition("2022-01-02T12:00:00")) .longValue()); - Assert.assertNull( + Assertions.assertNull( partitionOptimizedSequence.get( MixedDataTestHelpers.recordPartition("2022-01-03T12:00:00"))); - Assert.assertEquals( + Assertions.assertEquals( txId, partitionOptimizedSequence .get(MixedDataTestHelpers.recordPartition("2022-01-02T12:00:00")) @@ -177,7 +184,7 @@ public void testOverwritePartitionByExpression() { before, after, partitionOptimizedTime.get(MixedDataTestHelpers.recordPartition("2022-01-02T12:00:00"))); - Assert.assertNull( + Assertions.assertNull( partitionOptimizedTime.get(MixedDataTestHelpers.recordPartition("2022-01-03T12:00:00"))); assertRange( before, @@ -189,14 +196,14 @@ public void testOverwritePartitionByExpression() { getMixedTable().asKeyedTable(), Expressions.alwaysTrue()); // partition1 -> base[7,8,9] // partition3 -> base[3] - Assert.assertEquals(4, rows.size()); + Assertions.assertEquals(4, rows.size()); Set resultIdSet = Sets.newHashSet(); rows.forEach(r -> resultIdSet.add((Integer) r.get(0))); - Assert.assertTrue(resultIdSet.contains(7)); - Assert.assertTrue(resultIdSet.contains(8)); - Assert.assertTrue(resultIdSet.contains(9)); + Assertions.assertTrue(resultIdSet.contains(7)); + Assertions.assertTrue(resultIdSet.contains(8)); + Assertions.assertTrue(resultIdSet.contains(9)); - Assert.assertTrue(resultIdSet.contains(3)); + Assertions.assertTrue(resultIdSet.contains(3)); } } diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/op/TestRewritePartitions.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/op/TestRewritePartitions.java index a315ccbea7..5833827b18 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/op/TestRewritePartitions.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/op/TestRewritePartitions.java @@ -27,14 +27,21 @@ import org.apache.iceberg.data.Record; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.util.StructLikeMap; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import java.io.IOException; import java.util.List; import java.util.Set; public class TestRewritePartitions extends TableDataTestBase { + @BeforeEach + public void setUp() throws IOException { + initData(); + } + /** overwrite partition by data file. */ @Test public void testDynamicOverwritePartition() { @@ -56,18 +63,18 @@ public void testDynamicOverwritePartition() { StructLikeMap partitionOptimizedSequence = MixedTableUtil.readOptimizedSequence(getMixedTable().asKeyedTable()); // expect result: 1 partition with new txId, 2,3 partition use old txId - Assert.assertEquals( + Assertions.assertEquals( txId, partitionOptimizedSequence .get(MixedDataTestHelpers.recordPartition("2022-01-01T12:00:00")) .longValue()); - Assert.assertNull( + Assertions.assertNull( partitionOptimizedSequence.get( MixedDataTestHelpers.recordPartition("2022-01-02T12:00:00"))); - Assert.assertNull( + Assertions.assertNull( partitionOptimizedSequence.get( MixedDataTestHelpers.recordPartition("2022-01-03T12:00:00"))); - Assert.assertNull( + Assertions.assertNull( partitionOptimizedSequence.get( MixedDataTestHelpers.recordPartition("2022-01-04T12:00:00"))); @@ -77,14 +84,14 @@ public void testDynamicOverwritePartition() { // partition1 -> base[7,8,9] // partition2 -> base[2] // partition3 -> base[3] - Assert.assertEquals(5, rows.size()); + Assertions.assertEquals(5, rows.size()); Set resultIdSet = Sets.newHashSet(); rows.forEach(r -> resultIdSet.add((Integer) r.get(0))); - Assert.assertTrue(resultIdSet.contains(7)); - Assert.assertTrue(resultIdSet.contains(8)); - Assert.assertTrue(resultIdSet.contains(9)); - Assert.assertTrue(resultIdSet.contains(2)); - Assert.assertTrue(resultIdSet.contains(3)); + Assertions.assertTrue(resultIdSet.contains(7)); + Assertions.assertTrue(resultIdSet.contains(8)); + Assertions.assertTrue(resultIdSet.contains(9)); + Assertions.assertTrue(resultIdSet.contains(2)); + Assertions.assertTrue(resultIdSet.contains(3)); } } diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/op/TestUpdatePartitionProperties.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/op/TestUpdatePartitionProperties.java index b3926a13b2..41794df353 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/op/TestUpdatePartitionProperties.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/op/TestUpdatePartitionProperties.java @@ -26,15 +26,18 @@ import org.apache.iceberg.TestHelpers; import org.apache.iceberg.Transaction; import org.apache.iceberg.util.StructLikeMap; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import java.io.IOException; import java.util.Map; public class TestUpdatePartitionProperties extends TableTestBase { - public TestUpdatePartitionProperties() { - super( + @BeforeEach + public void setUp() throws IOException { + setupTable( new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), new BasicTableTestHelper(false, true)); } @@ -43,7 +46,7 @@ public TestUpdatePartitionProperties() { public void testUpdatePartitionProperties() { StructLikeMap> partitionProperties = getMixedTable().asUnkeyedTable().partitionProperty(); - Assert.assertEquals(0, partitionProperties.size()); + Assertions.assertEquals(0, partitionProperties.size()); StructLike p0 = TestHelpers.Row.of(1200); getMixedTable() .asUnkeyedTable() @@ -51,8 +54,8 @@ public void testUpdatePartitionProperties() { .set(p0, "key", "value") .commit(); partitionProperties = getMixedTable().asUnkeyedTable().partitionProperty(); - Assert.assertEquals(1, partitionProperties.size()); - Assert.assertEquals("value", partitionProperties.get(p0).get("key")); + Assertions.assertEquals(1, partitionProperties.size()); + Assertions.assertEquals("value", partitionProperties.get(p0).get("key")); } @Test @@ -60,7 +63,7 @@ public void testUpdatePartitionPropertiesInTx() { StructLikeMap> partitionProperties = getMixedTable().asUnkeyedTable().partitionProperty(); Transaction transaction = getMixedTable().asUnkeyedTable().newTransaction(); - Assert.assertEquals(0, partitionProperties.size()); + Assertions.assertEquals(0, partitionProperties.size()); StructLike p0 = TestHelpers.Row.of(1200); getMixedTable() .asUnkeyedTable() @@ -68,18 +71,18 @@ public void testUpdatePartitionPropertiesInTx() { .set(p0, "key", "value") .commit(); partitionProperties = getMixedTable().asUnkeyedTable().partitionProperty(); - Assert.assertEquals(0, partitionProperties.size()); + Assertions.assertEquals(0, partitionProperties.size()); transaction.commitTransaction(); partitionProperties = getMixedTable().asUnkeyedTable().partitionProperty(); - Assert.assertEquals(1, partitionProperties.size()); - Assert.assertEquals("value", partitionProperties.get(p0).get("key")); + Assertions.assertEquals(1, partitionProperties.size()); + Assertions.assertEquals("value", partitionProperties.get(p0).get("key")); } @Test public void testRemovePartitionProperties() { StructLikeMap> partitionProperties = getMixedTable().asUnkeyedTable().partitionProperty(); - Assert.assertEquals(0, partitionProperties.size()); + Assertions.assertEquals(0, partitionProperties.size()); StructLike p0 = TestHelpers.Row.of(1200); getMixedTable() .asUnkeyedTable() @@ -87,10 +90,10 @@ public void testRemovePartitionProperties() { .set(p0, "key", "value") .commit(); partitionProperties = getMixedTable().asUnkeyedTable().partitionProperty(); - Assert.assertEquals(1, partitionProperties.get(p0).size()); + Assertions.assertEquals(1, partitionProperties.get(p0).size()); getMixedTable().asUnkeyedTable().updatePartitionProperties(null).remove(p0, "key").commit(); partitionProperties = getMixedTable().asUnkeyedTable().partitionProperty(); - Assert.assertEquals(0, partitionProperties.get(p0).size()); + Assertions.assertEquals(0, partitionProperties.get(p0).size()); } } diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/op/TestUpdateTable.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/op/TestUpdateTable.java index 420c16b7af..793fe73320 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/op/TestUpdateTable.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/op/TestUpdateTable.java @@ -29,15 +29,18 @@ import org.apache.iceberg.UpdateProperties; import org.apache.iceberg.UpdateSchema; import org.apache.iceberg.types.Types; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import java.io.IOException; import java.util.Map; public class TestUpdateTable extends TableTestBase { - public TestUpdateTable() { - super( + @BeforeEach + public void setUp() throws IOException { + setupTable( new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), new BasicTableTestHelper(true, true)); } @@ -51,25 +54,25 @@ public void testUpdateProperties() throws TException { final String testProps2 = "test.props2"; final String testPropsVal2 = "v2"; - Assert.assertFalse(properties.containsKey(testProps)); - Assert.assertFalse(properties.containsKey(testProps2)); + Assertions.assertFalse(properties.containsKey(testProps)); + Assertions.assertFalse(properties.containsKey(testProps2)); UpdateProperties updateProperties = getMixedTable().updateProperties(); updateProperties.set(testProps, testPropsValue); updateProperties.commit(); properties = getMixedTable().properties(); - Assert.assertTrue(properties.containsKey(testProps)); - Assert.assertEquals(testPropsValue, properties.get(testProps)); + Assertions.assertTrue(properties.containsKey(testProps)); + Assertions.assertEquals(testPropsValue, properties.get(testProps)); updateProperties = getMixedTable().updateProperties(); updateProperties.remove(testProps); updateProperties.set(testProps2, testPropsVal2); updateProperties.commit(); properties = getMixedTable().properties(); - Assert.assertTrue(properties.containsKey(testProps2)); - Assert.assertEquals(testPropsVal2, properties.get(testProps2)); - Assert.assertFalse(properties.containsKey(testProps)); + Assertions.assertTrue(properties.containsKey(testProps2)); + Assertions.assertEquals(testPropsVal2, properties.get(testProps2)); + Assertions.assertFalse(properties.containsKey(testProps)); } @Test @@ -165,9 +168,9 @@ public void testSyncSchema() { us.commit(); KeyedSchemaUpdate.syncSchema(getMixedTable().asKeyedTable()); - Assert.assertEquals( - "Should match base and change schema", + Assertions.assertEquals( getMixedTable().asKeyedTable().baseTable().schema().asStruct(), - getMixedTable().asKeyedTable().changeTable().schema().asStruct()); + getMixedTable().asKeyedTable().changeTable().schema().asStruct(), + "Should match base and change schema"); } } diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/optimizing/IcebergRewriteExecutorTest.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/optimizing/IcebergRewriteExecutorTest.java index 37b79036cb..083dc7be54 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/optimizing/IcebergRewriteExecutorTest.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/optimizing/IcebergRewriteExecutorTest.java @@ -49,12 +49,11 @@ import org.apache.iceberg.parquet.Parquet; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.util.Pair; -import org.junit.Assert; -import org.junit.Assume; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; import java.util.Arrays; @@ -62,11 +61,11 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Stream; -@RunWith(Parameterized.class) public class IcebergRewriteExecutorTest extends TableTestBase { - private final FileFormat fileFormat; + private FileFormat fileFormat; private RewriteFilesInput scanTask; @@ -75,20 +74,14 @@ public class IcebergRewriteExecutorTest extends TableTestBase { private final Schema posSchema = new Schema(MetadataColumns.FILE_PATH, MetadataColumns.ROW_POSITION); - public IcebergRewriteExecutorTest(boolean hasPartition, FileFormat fileFormat) { - super( - new BasicCatalogTestHelper(TableFormat.ICEBERG), - new BasicTableTestHelper(false, true, buildTableProperties(fileFormat))); - this.fileFormat = fileFormat; - } - - @Parameterized.Parameters(name = "partitionedTable = {0}, fileFormat = {1}") - public static Object[][] parameters() { - return new Object[][] { - {true, FileFormat.PARQUET}, {false, FileFormat.PARQUET}, - {true, FileFormat.AVRO}, {false, FileFormat.AVRO}, - {true, FileFormat.ORC}, {false, FileFormat.ORC} - }; + public static Stream parameters() { + return Stream.of( + Arguments.of(true, FileFormat.PARQUET), + Arguments.of(false, FileFormat.PARQUET), + Arguments.of(true, FileFormat.AVRO), + Arguments.of(false, FileFormat.AVRO), + Arguments.of(true, FileFormat.ORC), + Arguments.of(false, FileFormat.ORC)); } private static Map buildTableProperties(FileFormat fileFormat) { @@ -99,6 +92,14 @@ private static Map buildTableProperties(FileFormat fileFormat) { return tableProperties; } + private void prepare(boolean hasPartition, FileFormat fileFormat) throws IOException { + setupTable( + new BasicCatalogTestHelper(TableFormat.ICEBERG), + new BasicTableTestHelper(false, true, buildTableProperties(fileFormat))); + this.fileFormat = fileFormat; + initDataAndReader(); + } + private StructLike getPartitionData() { if (isPartitionedTable()) { return TestHelpers.Row.of(0); @@ -107,8 +108,7 @@ private StructLike getPartitionData() { } } - @Before - public void initDataAndReader() throws IOException { + private void initDataAndReader() throws IOException { StructLike partitionData = getPartitionData(); OutputFileFactory outputFileFactory = OutputFileFactory.builderFor(getMixedTable().asUnkeyedTable(), 0, 1) @@ -164,8 +164,14 @@ public void initDataAndReader() throws IOException { getMixedTable()); } - @Test - public void readAllData() throws IOException { + @ParameterizedTest(name = "partitionedTable = {0}, fileFormat = {1}") + @MethodSource("parameters") + public void readAllData(boolean hasPartition, FileFormat fileFormat) throws IOException { + prepare(hasPartition, fileFormat); + runReadAllData(); + } + + private void runReadAllData() throws IOException { IcebergRewriteExecutor executor = new IcebergRewriteExecutor(scanTask, getMixedTable(), Collections.emptyMap()); @@ -177,9 +183,9 @@ public void readAllData() throws IOException { output.getDataFiles()[0].format(), getMixedTable().schema(), new HashMap<>())) { - Assert.assertEquals(1, Iterables.size(records)); + Assertions.assertEquals(1, Iterables.size(records)); Record record = Iterables.getFirst(records, null); - Assert.assertEquals(record.get(0), 3); + Assertions.assertEquals(record.get(0), 3); } try (CloseableIterable records = @@ -188,28 +194,33 @@ public void readAllData() throws IOException { output.getDataFiles()[0].format(), posSchema, new HashMap<>())) { - Assert.assertEquals(2, Iterables.size(records)); + Assertions.assertEquals(2, Iterables.size(records)); Record first = Iterables.getFirst(records, null); - Assert.assertEquals(first.get(1), 0L); + Assertions.assertEquals(first.get(1), 0L); Record last = Iterables.getLast(records); - Assert.assertEquals(last.get(1), 1L); + Assertions.assertEquals(last.get(1), 1L); } } - @Test - public void readAllDataWithPartitionEvolution() throws IOException { - Assume.assumeTrue(getMixedTable().spec().isPartitioned()); + @ParameterizedTest(name = "partitionedTable = {0}, fileFormat = {1}") + @MethodSource("parameters") + public void readAllDataWithPartitionEvolution(boolean hasPartition, FileFormat fileFormat) + throws IOException { + prepare(hasPartition, fileFormat); + Assumptions.assumeTrue(getMixedTable().spec().isPartitioned()); getMixedTable() .asUnkeyedTable() .updateSpec() .removeField("op_time_day") .addField(Expressions.month("op_time")) .commit(); - readAllData(); + runReadAllData(); } - @Test - public void readOnlyData() throws IOException { + @ParameterizedTest(name = "partitionedTable = {0}, fileFormat = {1}") + @MethodSource("parameters") + public void readOnlyData(boolean hasPartition, FileFormat fileFormat) throws IOException { + prepare(hasPartition, fileFormat); IcebergRewriteExecutor executor = new IcebergRewriteExecutor(dataScanTask, getMixedTable(), Collections.emptyMap()); @@ -221,10 +232,10 @@ public void readOnlyData() throws IOException { output.getDataFiles()[0].format(), getMixedTable().schema(), new HashMap<>())) { - Assert.assertEquals(3, Iterables.size(records)); + Assertions.assertEquals(3, Iterables.size(records)); } - Assert.assertTrue(output.getDeleteFiles() == null || output.getDeleteFiles().length == 0); + Assertions.assertTrue(output.getDeleteFiles() == null || output.getDeleteFiles().length == 0); } private CloseableIterable openFile( diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/optimizing/evaluation/TestMetadataBasedEvaluationEvent.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/optimizing/evaluation/TestMetadataBasedEvaluationEvent.java index 49229d4813..b8dd2012e7 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/optimizing/evaluation/TestMetadataBasedEvaluationEvent.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/optimizing/evaluation/TestMetadataBasedEvaluationEvent.java @@ -62,10 +62,10 @@ import org.apache.iceberg.io.WriteResult; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.Pair; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; import java.io.UncheckedIOException; @@ -75,15 +75,10 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import java.util.stream.Stream; -@RunWith(Parameterized.class) public class TestMetadataBasedEvaluationEvent extends TableTestBase { - public TestMetadataBasedEvaluationEvent( - CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) { - super(catalogTestHelper, tableTestHelper); - } - public static final Schema TABLE_SCHEMA = new Schema( Lists.newArrayList( @@ -95,15 +90,14 @@ public TestMetadataBasedEvaluationEvent( public static final PartitionSpec SPEC = PartitionSpec.builderFor(TABLE_SCHEMA).day("op_time").build(); - @Parameterized.Parameters(name = "{0}, {1}") - public static Object[] parameters() { - return new Object[][] { - { - new BasicCatalogTestHelper(TableFormat.ICEBERG), - new BasicTableTestHelper(TABLE_SCHEMA, true, SPEC) - }, - {new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), new BasicTableTestHelper(true, true)} - }; + public static Stream parameters() { + return Stream.of( + Arguments.of( + new BasicCatalogTestHelper(TableFormat.ICEBERG), + new BasicTableTestHelper(TABLE_SCHEMA, true, SPEC)), + Arguments.of( + new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), + new BasicTableTestHelper(true, true))); } public void initData() throws IOException { @@ -216,17 +210,23 @@ public void addChangeStoreData() throws IOException { } } - @Test - public void test_evaluating_metadataBasedTriggerEnabled() { + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("parameters") + public void test_evaluating_metadataBasedTriggerEnabled( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) throws IOException { + setupTable(catalogTestHelper, tableTestHelper); OptimizingConfig config = getDefaultOptimizingConfig(); - Assert.assertFalse(config.isMetadataBasedTriggerEnabled()); + Assertions.assertFalse(config.isMetadataBasedTriggerEnabled()); config.setEvaluationFallbackInterval(Long.MAX_VALUE); - Assert.assertTrue(config.isMetadataBasedTriggerEnabled()); + Assertions.assertTrue(config.isMetadataBasedTriggerEnabled()); } - @Test - public void test_evaluating_emptyTable() { + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("parameters") + public void test_evaluating_emptyTable( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) throws IOException { + setupTable(catalogTestHelper, tableTestHelper); // Temporarily set self-optimizing.evaluation.fallback-interval to Long.MAX_VALUE to prevent // triggering due to reaching the fallback interval. OptimizingConfig config = @@ -236,18 +236,21 @@ public void test_evaluating_emptyTable() { // Verify the empty table stats TableStatsProvider.BasicFileStats stats = MixedAndIcebergTableStatsProvider.INSTANCE.collect(table); - Assert.assertEquals(0, stats.dataFileCnt); - Assert.assertEquals(0, stats.totalFileSize); - Assert.assertEquals(0, stats.deleteFileCnt); + Assertions.assertEquals(0, stats.dataFileCnt); + Assertions.assertEquals(0, stats.totalFileSize); + Assertions.assertEquals(0, stats.deleteFileCnt); // Verify the empty table evaluation - Assert.assertTrue(config.isMetadataBasedTriggerEnabled()); - Assert.assertFalse(MetadataBasedEvaluationEvent.isReachFallbackInterval(config, 0L)); - Assert.assertFalse(MetadataBasedEvaluationEvent.isEvaluatingNecessary(config, table, 0L)); + Assertions.assertTrue(config.isMetadataBasedTriggerEnabled()); + Assertions.assertFalse(MetadataBasedEvaluationEvent.isReachFallbackInterval(config, 0L)); + Assertions.assertFalse(MetadataBasedEvaluationEvent.isEvaluatingNecessary(config, table, 0L)); } - @Test - public void test_evaluating_nonEmptyTable() throws IOException { + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("parameters") + public void test_evaluating_nonEmptyTable( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) throws IOException { + setupTable(catalogTestHelper, tableTestHelper); initData(); OptimizingConfig config = getDefaultOptimizingConfig(); MixedTable table = getMixedTable(); @@ -255,39 +258,42 @@ public void test_evaluating_nonEmptyTable() throws IOException { // Verify the nonEmpty table stats TableStatsProvider.BasicFileStats stats = MixedAndIcebergTableStatsProvider.INSTANCE.collect(table); - Assert.assertTrue(stats.dataFileCnt > 0); - Assert.assertTrue(stats.totalFileSize > 0); - Assert.assertEquals(0, stats.deleteFileCnt); + Assertions.assertTrue(stats.dataFileCnt > 0); + Assertions.assertTrue(stats.totalFileSize > 0); + Assertions.assertEquals(0, stats.deleteFileCnt); // 1. Test for metadata-based trigger disabled. config.setEvaluationFallbackInterval(-1); - Assert.assertFalse(config.isMetadataBasedTriggerEnabled()); + Assertions.assertFalse(config.isMetadataBasedTriggerEnabled()); // 2. Test for metadata-based trigger enabled. // Temporarily set self-optimizing.evaluation.fallback-interval to Long.MAX_VALUE to prevent // triggering due to reaching the fallback interval. config.setEvaluationFallbackInterval(Long.MAX_VALUE); - Assert.assertTrue(config.isMetadataBasedTriggerEnabled()); - Assert.assertFalse(MetadataBasedEvaluationEvent.isReachFallbackInterval(config, 0L)); + Assertions.assertTrue(config.isMetadataBasedTriggerEnabled()); + Assertions.assertFalse(MetadataBasedEvaluationEvent.isReachFallbackInterval(config, 0L)); // 2.1 Test for evaluating pendingInput necessary. config.setTargetSize(134217728); - Assert.assertTrue(MetadataBasedEvaluationEvent.isEvaluatingNecessary(config, table, 0L)); + Assertions.assertTrue(MetadataBasedEvaluationEvent.isEvaluatingNecessary(config, table, 0L)); // 2.2 Test for evaluating pendingInput not necessary // Temporarily set self-optimizing.target-size to a lower value than the average file size config.setTargetSize(100); - Assert.assertFalse(MetadataBasedEvaluationEvent.isEvaluatingNecessary(config, table, 0L)); + Assertions.assertFalse(MetadataBasedEvaluationEvent.isEvaluatingNecessary(config, table, 0L)); // 2.3 Test for evaluating pendingInput necessary because the fallback interval has been // reached. config.setEvaluationFallbackInterval(0L); - Assert.assertTrue(config.isMetadataBasedTriggerEnabled()); - Assert.assertTrue(MetadataBasedEvaluationEvent.isEvaluatingNecessary(config, table, 0L)); + Assertions.assertTrue(config.isMetadataBasedTriggerEnabled()); + Assertions.assertTrue(MetadataBasedEvaluationEvent.isEvaluatingNecessary(config, table, 0L)); } - @Test - public void test_evaluating_pendingInput_nonEmptyTable() throws IOException { + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("parameters") + public void test_evaluating_pendingInput_nonEmptyTable( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) throws IOException { + setupTable(catalogTestHelper, tableTestHelper); initData(); // Set metadata-based trigger enabled and fallback interval not reached. OptimizingConfig config = @@ -300,51 +306,51 @@ public void test_evaluating_pendingInput_nonEmptyTable() throws IOException { Map partitionPlanMap = initPartitionPlans(tableFileScanHelper, config); - Assert.assertEquals(1, partitionPlanMap.size()); + Assertions.assertEquals(1, partitionPlanMap.size()); long sizeSquaredErrorSum = ((CommonPartitionEvaluator) new ArrayList<>(partitionPlanMap.values()).get(0)) .getFileSizeSquaredErrorSum(); - Assert.assertEquals(0L, sizeSquaredErrorSum); + Assertions.assertEquals(0L, sizeSquaredErrorSum); List necessaryPartitions = partitionPlanMap.values().stream() .filter(PartitionEvaluator::isNecessary) .collect(Collectors.toList()); - Assert.assertEquals(1, necessaryPartitions.size()); + Assertions.assertEquals(1, necessaryPartitions.size()); // 2. Set mse tolerance > 0 to enabled file size square error sum update during initializing // Partition plans. config.setEvaluationMseTolerance(120000000); partitionPlanMap = initPartitionPlans(tableFileScanHelper, config); - Assert.assertEquals(1, partitionPlanMap.size()); + Assertions.assertEquals(1, partitionPlanMap.size()); sizeSquaredErrorSum = ((CommonPartitionEvaluator) new ArrayList<>(partitionPlanMap.values()).get(0)) .getFileSizeSquaredErrorSum(); - Assert.assertTrue(sizeSquaredErrorSum > 0); + Assertions.assertTrue(sizeSquaredErrorSum > 0); necessaryPartitions = partitionPlanMap.values().stream() .filter(PartitionEvaluator::isNecessary) .collect(Collectors.toList()); - Assert.assertEquals(0, necessaryPartitions.size()); + Assertions.assertEquals(0, necessaryPartitions.size()); // 3. Test the file size variance updated after adding change data. addChangeStoreData(); partitionPlanMap = initPartitionPlans(initTableFileScanHelper(), config); - Assert.assertEquals(1, partitionPlanMap.size()); + Assertions.assertEquals(1, partitionPlanMap.size()); long sizeSquaredErrorSumUpdated1 = ((CommonPartitionEvaluator) new ArrayList<>(partitionPlanMap.values()).get(0)) .getFileSizeSquaredErrorSum(); - Assert.assertNotEquals(sizeSquaredErrorSum, sizeSquaredErrorSumUpdated1); + Assertions.assertNotEquals(sizeSquaredErrorSum, sizeSquaredErrorSumUpdated1); necessaryPartitions = partitionPlanMap.values().stream() .filter(PartitionEvaluator::isNecessary) .collect(Collectors.toList()); - Assert.assertEquals(0, necessaryPartitions.size()); + Assertions.assertEquals(0, necessaryPartitions.size()); // 4. Set mse tolerance smaller for partitions to test necessary pending. config.setEvaluationMseTolerance(100000000); @@ -353,13 +359,13 @@ public void test_evaluating_pendingInput_nonEmptyTable() throws IOException { long sizeSquaredErrorSumUpdated2 = ((CommonPartitionEvaluator) new ArrayList<>(partitionPlanMap.values()).get(0)) .getFileSizeSquaredErrorSum(); - Assert.assertEquals(sizeSquaredErrorSumUpdated2, sizeSquaredErrorSumUpdated1); + Assertions.assertEquals(sizeSquaredErrorSumUpdated2, sizeSquaredErrorSumUpdated1); necessaryPartitions = partitionPlanMap.values().stream() .filter(PartitionEvaluator::isNecessary) .collect(Collectors.toList()); - Assert.assertEquals(1, necessaryPartitions.size()); + Assertions.assertEquals(1, necessaryPartitions.size()); } private TableFileScanHelper initTableFileScanHelper() { @@ -448,35 +454,41 @@ private PartitionEvaluator buildEvaluator( } } - @Test - public void test_setFileSizeMSETolerance() { + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("parameters") + public void test_setFileSizeMSETolerance( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) throws IOException { + setupTable(catalogTestHelper, tableTestHelper); OptimizingConfig config = new OptimizingConfig(); - Assert.assertEquals(0, config.getEvaluationMseTolerance()); + Assertions.assertEquals(0, config.getEvaluationMseTolerance()); config.setEvaluationMseTolerance(1000); - Assert.assertEquals(1000, config.getEvaluationMseTolerance()); + Assertions.assertEquals(1000, config.getEvaluationMseTolerance()); config.setEvaluationMseTolerance(140000000); - Assert.assertEquals(140000000, config.getEvaluationMseTolerance()); + Assertions.assertEquals(140000000, config.getEvaluationMseTolerance()); } - @Test - public void testBasicStatsAccept() { + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("parameters") + public void testBasicStatsAccept( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) throws IOException { + setupTable(catalogTestHelper, tableTestHelper); MixedAndIcebergTableStatsProvider.BasicFileStats stats = new MixedAndIcebergTableStatsProvider.BasicFileStats(); // Initial state should be zeros - Assert.assertEquals(0, stats.deleteFileCnt); - Assert.assertEquals(0, stats.dataFileCnt); - Assert.assertEquals(0, stats.totalFileSize); + Assertions.assertEquals(0, stats.deleteFileCnt); + Assertions.assertEquals(0, stats.dataFileCnt); + Assertions.assertEquals(0, stats.totalFileSize); // Create first summary map Map summary1 = new HashMap<>(); summary1.put("total-delete-files", "5"); summary1.put("total-data-files", "10"); summary1.put("total-files-size", "1024"); stats.accept(summary1); - Assert.assertEquals(5, stats.deleteFileCnt); - Assert.assertEquals(10, stats.dataFileCnt); - Assert.assertEquals(1024, stats.totalFileSize); + Assertions.assertEquals(5, stats.deleteFileCnt); + Assertions.assertEquals(10, stats.dataFileCnt); + Assertions.assertEquals(1024, stats.totalFileSize); // Create second summary map to test accumulation Map summary2 = new HashMap<>(); summary2.put("total-delete-files", "3"); @@ -484,13 +496,16 @@ public void testBasicStatsAccept() { summary2.put("total-files-size", "2048"); stats.accept(summary2); // Values should be accumulated - Assert.assertEquals(8, stats.deleteFileCnt); // 5 + 3 - Assert.assertEquals(17, stats.dataFileCnt); // 10 + 7 - Assert.assertEquals(3072, stats.totalFileSize); // 1024 + 2048 + Assertions.assertEquals(8, stats.deleteFileCnt); // 5 + 3 + Assertions.assertEquals(17, stats.dataFileCnt); // 10 + 7 + Assertions.assertEquals(3072, stats.totalFileSize); // 1024 + 2048 } - @Test - public void testBasicTableStatsAcceptWithMissingProperties() { + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("parameters") + public void testBasicTableStatsAcceptWithMissingProperties( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) throws IOException { + setupTable(catalogTestHelper, tableTestHelper); MixedAndIcebergTableStatsProvider.BasicFileStats stats = new MixedAndIcebergTableStatsProvider.BasicFileStats(); // Summary map with missing properties should use default values @@ -498,13 +513,16 @@ public void testBasicTableStatsAcceptWithMissingProperties() { // Only provide one property, others should default to 0 summary.put("total-data-files", "15"); stats.accept(summary); - Assert.assertEquals(0, stats.deleteFileCnt); // default value - Assert.assertEquals(15, stats.dataFileCnt); - Assert.assertEquals(0, stats.totalFileSize); // default value + Assertions.assertEquals(0, stats.deleteFileCnt); // default value + Assertions.assertEquals(15, stats.dataFileCnt); + Assertions.assertEquals(0, stats.totalFileSize); // default value } - @Test - public void testBasicTableStatsAcceptWithInvalidValues() { + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("parameters") + public void testBasicTableStatsAcceptWithInvalidValues( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) throws IOException { + setupTable(catalogTestHelper, tableTestHelper); MixedAndIcebergTableStatsProvider.BasicFileStats stats = new MixedAndIcebergTableStatsProvider.BasicFileStats(); // Summary map with invalid numeric values should use default values @@ -514,7 +532,7 @@ public void testBasicTableStatsAcceptWithInvalidValues() { summary.put("total-files-size", "not-a-number"); // Invalid values should throw NumberFormatException - Assert.assertThrows(NumberFormatException.class, () -> stats.accept(summary)); + Assertions.assertThrows(NumberFormatException.class, () -> stats.accept(summary)); } OptimizingConfig getDefaultOptimizingConfig() { diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/scan/TestBaseCombinedScanTask.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/scan/TestBaseCombinedScanTask.java index 4aadb9c519..3bc2e69b5a 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/scan/TestBaseCombinedScanTask.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/scan/TestBaseCombinedScanTask.java @@ -24,8 +24,8 @@ import org.apache.iceberg.DataFile; import org.apache.iceberg.DataFiles; import org.apache.iceberg.PartitionSpec; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.util.Collections; @@ -62,6 +62,6 @@ public void testToString() { + "\t\tfileSizeInBytes=10, \n" + "\t\trecordCount=100}], \n" + "\tdeleteFiles=[]}}"; - Assert.assertEquals(expected, baseCombinedScanTask.toString()); + Assertions.assertEquals(expected, baseCombinedScanTask.toString()); } } diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/scan/TestKeyedTableScan.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/scan/TestKeyedTableScan.java index 118718fc37..5cac19d212 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/scan/TestKeyedTableScan.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/scan/TestKeyedTableScan.java @@ -28,8 +28,9 @@ import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.io.WriteResult; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.ArrayList; @@ -38,6 +39,11 @@ public class TestKeyedTableScan extends TableDataTestBase { + @BeforeEach + public void setUp() throws IOException { + initData(); + } + @Test public void testScanWithInsertFileInBaseStore() throws IOException { assertFileCount(4, 2, 1); @@ -66,9 +72,9 @@ private void assertFileCount(int baseFileCnt, int insertFileCnt, int equDeleteFi }); } } - Assert.assertEquals(baseFileCnt, allBaseTasks.size()); - Assert.assertEquals(insertFileCnt, allInsertTasks.size()); - Assert.assertEquals(equDeleteFileCnt, allEquDeleteTasks.size()); + Assertions.assertEquals(baseFileCnt, allBaseTasks.size()); + Assertions.assertEquals(insertFileCnt, allInsertTasks.size()); + Assertions.assertEquals(equDeleteFileCnt, allEquDeleteTasks.size()); } private void writeInsertFileIntoBaseStore() throws IOException { diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/scan/TestMixedChangeTableScan.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/scan/TestMixedChangeTableScan.java index 48563aad28..6fd30e67f0 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/scan/TestMixedChangeTableScan.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/scan/TestMixedChangeTableScan.java @@ -24,14 +24,20 @@ import org.apache.iceberg.StructLike; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.util.StructLikeMap; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.function.Predicate; public class TestMixedChangeTableScan extends TableDataTestBase { + @BeforeEach + public void setUp() throws IOException { + initData(); + } + @Test public void testIncrementalScan() throws IOException { ChangeTableIncrementalScan changeTableIncrementalScan = @@ -95,10 +101,10 @@ private void assertFiles( int taskCount = 0; for (FileScanTask task : tasks) { taskCount++; - Assert.assertTrue(task instanceof BasicMixedFileScanTask); - Assert.assertTrue(validator.test(task)); + Assertions.assertTrue(task instanceof BasicMixedFileScanTask); + Assertions.assertTrue(validator.test(task)); } - Assert.assertEquals(fileCnt, taskCount); + Assertions.assertEquals(fileCnt, taskCount); } private void assertFilesSequence( diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/scan/TestNodeFileScanTask.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/scan/TestNodeFileScanTask.java index ab371be239..164b748488 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/scan/TestNodeFileScanTask.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/scan/TestNodeFileScanTask.java @@ -24,8 +24,8 @@ import org.apache.iceberg.DataFile; import org.apache.iceberg.DataFiles; import org.apache.iceberg.PartitionSpec; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.util.Arrays; import java.util.List; @@ -75,7 +75,7 @@ public void testDataTasksReturnsSameInstanceOnConsecutiveCalls() { List first = nodeTask.dataTasks(); List second = nodeTask.dataTasks(); - Assert.assertSame("dataTasks() should return the same cached instance", first, second); + Assertions.assertSame(first, second, "dataTasks() should return the same cached instance"); } @Test @@ -87,16 +87,16 @@ public void testDataTasksCacheIsInvalidatedAfterAddFile() { nodeTask.addFile(baseTask); List before = nodeTask.dataTasks(); - Assert.assertEquals("Before addFile: should have 1 data task", 1, before.size()); + Assertions.assertEquals(1, before.size(), "Before addFile: should have 1 data task"); MixedFileScanTask insertTask = createInsertTask("/tmp/1-I-3-00000-0-9009257362994691056-00002.parquet"); nodeTask.addFile(insertTask); List after = nodeTask.dataTasks(); - Assert.assertEquals("After addFile: should have 2 data tasks", 2, after.size()); - Assert.assertNotSame( - "dataTasks() should return a new instance after addFile invalidates cache", before, after); + Assertions.assertEquals(2, after.size(), "After addFile: should have 2 data tasks"); + Assertions.assertNotSame( + before, after, "dataTasks() should return a new instance after addFile invalidates cache"); } @Test @@ -108,7 +108,7 @@ public void testDataTasksIsUnmodifiable() { List tasks = nodeTask.dataTasks(); try { tasks.add(baseTask); - Assert.fail("dataTasks() should return an unmodifiable list"); + Assertions.fail("dataTasks() should return an unmodifiable list"); } catch (UnsupportedOperationException e) { // expected } @@ -126,10 +126,10 @@ public void testDataTasksContainsBothBaseAndInsertTasks() { nodeTask.addFile(insertTask); List tasks = nodeTask.dataTasks(); - Assert.assertEquals( - "dataTasks() should contain both baseTasks and insertTasks", 2, tasks.size()); - Assert.assertTrue("dataTasks() should include the base task", tasks.contains(baseTask)); - Assert.assertTrue("dataTasks() should include the insert task", tasks.contains(insertTask)); + Assertions.assertEquals( + 2, tasks.size(), "dataTasks() should contain both baseTasks and insertTasks"); + Assertions.assertTrue(tasks.contains(baseTask), "dataTasks() should include the base task"); + Assertions.assertTrue(tasks.contains(insertTask), "dataTasks() should include the insert task"); } @Test @@ -145,7 +145,8 @@ public void testDataTasksDoesNotContainEqDeleteFiles() { nodeTask.addFile(deleteTask); List tasks = nodeTask.dataTasks(); - Assert.assertEquals("dataTasks() should not include EQ_DELETE_FILE tasks", 1, tasks.size()); - Assert.assertTrue("dataTasks() should include only the base task", tasks.contains(baseTask)); + Assertions.assertEquals(1, tasks.size(), "dataTasks() should not include EQ_DELETE_FILE tasks"); + Assertions.assertTrue( + tasks.contains(baseTask), "dataTasks() should include only the base task"); } } diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/scan/TestScanSplitTask.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/scan/TestScanSplitTask.java index 25d6d75c2b..fb9942cd46 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/scan/TestScanSplitTask.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/scan/TestScanSplitTask.java @@ -27,13 +27,20 @@ import org.apache.iceberg.data.Record; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.WriteResult; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.Arrays; public class TestScanSplitTask extends TableDataTestBase { + + @BeforeEach + public void setUp() throws IOException { + initData(); + } + @Test public void testSplitTaskByDeleteRatio() throws IOException { writeInsertFileIntoBaseStore(); @@ -42,7 +49,7 @@ public void testSplitTaskByDeleteRatio() throws IOException { CloseableIterable combinedScanTasks = getMixedTable().asKeyedTable().newScan().planTasks(); for (CombinedScanTask combinedScanTask : combinedScanTasks) { - Assert.assertEquals(combinedScanTask.tasks().size(), 7); + Assertions.assertEquals(combinedScanTask.tasks().size(), 7); } } { @@ -56,7 +63,7 @@ public void testSplitTaskByDeleteRatio() throws IOException { // delete content. Therefore, after enableSplitTaskByDeleteRatio is turned on, two // additional tasks will be // split out. - Assert.assertEquals(combinedScanTask.tasks().size(), 9); + Assertions.assertEquals(combinedScanTask.tasks().size(), 9); } } } diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/scan/TestTableEntriesScan.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/scan/TestTableEntriesScan.java index f261baa88b..ec9c922933 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/scan/TestTableEntriesScan.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/scan/TestTableEntriesScan.java @@ -38,8 +38,9 @@ import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.WriteResult; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.Arrays; @@ -49,6 +50,11 @@ public class TestTableEntriesScan extends TableDataTestBase { + @BeforeEach + public void setUp() throws IOException { + initData(); + } + @Test public void testScanEntriesForDataFile() { // change table commit 2 insert files, then commit 1 delete file @@ -61,7 +67,7 @@ public void testScanEntriesForDataFile() { cnt++; assertEntry(expectedEntries, entry); } - Assert.assertEquals(3, cnt); + Assertions.assertEquals(3, cnt); } @Test @@ -78,7 +84,7 @@ public void testScanEntriesForPosDeleteFiles() { cnt++; assertEntry(expectedEntries, entry); } - Assert.assertEquals(1, cnt); + Assertions.assertEquals(1, cnt); } @Test @@ -109,7 +115,7 @@ public void testScanAllEntries() throws IOException { cnt++; assertEntry(expectedEntries1, entry); } - Assert.assertEquals(5, cnt); + Assertions.assertEquals(5, cnt); // current TableEntriesScan entryScan2 = @@ -123,7 +129,7 @@ public void testScanAllEntries() throws IOException { cnt++; assertEntry(expectedEntries2, entry); } - Assert.assertEquals(9, cnt); + Assertions.assertEquals(9, cnt); // all entries (in all snapshots) TableEntriesScan entryScan3 = @@ -133,7 +139,7 @@ public void testScanAllEntries() throws IOException { .withAliveEntry(false) .allEntries() .build(); - Assert.assertEquals(13, Iterables.size(entryScan3.entries())); + Assertions.assertEquals(13, Iterables.size(entryScan3.entries())); } @Test @@ -151,7 +157,7 @@ public void testScanEntriesWithFilter() { cnt++; assertEntry(expectedEntries, entry); } - Assert.assertEquals(2, cnt); + Assertions.assertEquals(2, cnt); } @Test @@ -169,7 +175,7 @@ public void testScanEntriesFromSequence() throws IOException { assertEntry(expectedEntries, entry); } } - Assert.assertEquals(1, cnt); + Assertions.assertEquals(1, cnt); // base table commit 4 insert files, then commit 1 pos-delete file Table baseTable = getMixedTable().asKeyedTable().baseTable(); @@ -186,7 +192,7 @@ public void testScanEntriesFromSequence() throws IOException { assertEntry(expectedEntries, entry); } } - Assert.assertEquals(1, cnt); + Assertions.assertEquals(1, cnt); } @Test @@ -198,7 +204,7 @@ public void testScanEntriesWithPartitionSpecEvolution() throws IOException { // evolve partition spec: add identity(id) baseTable.updateSpec().addField("id").commit(); PartitionSpec newSpec = baseTable.spec(); - Assert.assertNotEquals(originalSpec.specId(), newSpec.specId()); + Assertions.assertNotEquals(originalSpec.specId(), newSpec.specId()); // append a data file under the new spec DataFile newSpecFile = @@ -222,18 +228,18 @@ public void testScanEntriesWithPartitionSpecEvolution() throws IOException { for (IcebergFileEntry entry : entries) { count++; ContentFile file = entry.getFile(); - Assert.assertNotNull(file.partition()); + Assertions.assertNotNull(file.partition()); // verify partition values are not null for partitioned files PartitionSpec fileSpec = baseTable.specs().get(file.specId()); if (fileSpec.isPartitioned()) { // op_time_day should be present for all specs - Assert.assertNotNull(file.partition().get(0, Object.class)); + Assertions.assertNotNull(file.partition().get(0, Object.class)); } } } // original: 4 data + 1 pos-delete + 1 new data = 6 - Assert.assertEquals(6, count); + Assertions.assertEquals(6, count); } private List writeIntoBase() throws IOException { @@ -255,12 +261,13 @@ private List writeIntoBase() throws IOException { private void assertEntry(Map expected, IcebergFileEntry entry) { Entry expectEntry = expected.get(entry.getFile().path().toString()); - Assert.assertNotNull(expectEntry); - Assert.assertEquals( - "fail file " + entry, expectEntry.getSequenceNumber(), entry.getSequenceNumber()); - Assert.assertEquals("fail file " + entry, expectEntry.getSnapshotId(), entry.getSnapshotId()); - Assert.assertEquals( - "fail file " + entry, expectEntry.getFileContent(), entry.getFile().content()); + Assertions.assertNotNull(expectEntry); + Assertions.assertEquals( + expectEntry.getSequenceNumber(), entry.getSequenceNumber(), "fail file " + entry); + Assertions.assertEquals( + expectEntry.getSnapshotId(), entry.getSnapshotId(), "fail file " + entry); + Assertions.assertEquals( + expectEntry.getFileContent(), entry.getFile().content(), "fail file " + entry); if (expectEntry.isAliveEntry()) { assertAliveEntry(entry); } else { @@ -334,13 +341,14 @@ public FileContent getFileContent() { } private void assertAliveEntry(IcebergFileEntry entry) { - Assert.assertTrue( - "fail file " + entry, + Assertions.assertTrue( entry.getStatus() == ManifestEntryFields.Status.ADDED - || entry.getStatus() == ManifestEntryFields.Status.EXISTING); + || entry.getStatus() == ManifestEntryFields.Status.EXISTING, + "fail file " + entry); } private void assertDeletedEntry(IcebergFileEntry entry) { - Assert.assertSame("fail file " + entry, entry.getStatus(), ManifestEntryFields.Status.DELETED); + Assertions.assertSame( + entry.getStatus(), ManifestEntryFields.Status.DELETED, "fail file " + entry); } } diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/table/blocker/TestBasicTableBlockerManager.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/table/blocker/TestBasicTableBlockerManager.java index 267dbe6f91..83c5a9adc5 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/table/blocker/TestBasicTableBlockerManager.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/table/blocker/TestBasicTableBlockerManager.java @@ -25,9 +25,11 @@ import org.apache.amoro.api.OperationConflictException; import org.apache.amoro.catalog.BasicCatalogTestHelper; import org.apache.amoro.catalog.TableTestBase; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -40,8 +42,9 @@ public class TestBasicTableBlockerManager extends TableTestBase { OPERATIONS.add(BlockableOperation.BATCH_WRITE); } - public TestBasicTableBlockerManager() { - super( + @BeforeEach + public void setUp() throws IOException { + setupTable( new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), new BasicTableTestHelper(true, true)); } @@ -50,13 +53,13 @@ public TestBasicTableBlockerManager() { public void testBlockAndRelease() throws OperationConflictException { TableBlockerManager tableBlockerManager = getMixedFormatCatalog().getTableBlockerManager(TableTestHelper.TEST_TABLE_ID); - Assert.assertTrue(tableBlockerManager instanceof BasicTableBlockerManager); + Assertions.assertTrue(tableBlockerManager instanceof BasicTableBlockerManager); BasicTableBlockerManager blockerManager = (BasicTableBlockerManager) tableBlockerManager; Blocker block = blockerManager.block(OPERATIONS); - Assert.assertTrue(block instanceof RenewableBlocker); + Assertions.assertTrue(block instanceof RenewableBlocker); blockerManager.release(block); - Assert.assertTrue(((RenewableBlocker) block).getRenewTaskFuture() == null); + Assertions.assertTrue(((RenewableBlocker) block).getRenewTaskFuture() == null); } } diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/trace/TestTableTracer.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/trace/TestTableTracer.java index 826e54f6d3..92421c7fb4 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/trace/TestTableTracer.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/trace/TestTableTracer.java @@ -45,44 +45,44 @@ import org.apache.iceberg.Snapshot; import org.apache.iceberg.Transaction; import org.apache.iceberg.data.Record; -import org.junit.Assert; -import org.junit.Assume; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.stream.Stream; /** Table trace is disabled for version 0.5.0 */ -@Ignore -@RunWith(Parameterized.class) +@Disabled public class TestTableTracer extends TableTestBase { - private final boolean onBaseTable; + private boolean onBaseTable; private UnkeyedTable operationTable; - public TestTableTracer(boolean keyedTable, boolean onBaseTable, boolean partitionedTable) { - super( + public static Stream parameters() { + return Stream.of( + Arguments.of(true, true, true), + Arguments.of(true, true, false), + Arguments.of(true, false, true), + Arguments.of(true, false, false), + Arguments.of(false, true, true), + Arguments.of(false, true, false)); + } + + private void prepare(boolean keyedTable, boolean onBaseTable, boolean partitionedTable) + throws IOException { + setupTable( new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), new BasicTableTestHelper(keyedTable, partitionedTable)); this.onBaseTable = onBaseTable; - } - - @Parameterized.Parameters(name = "keyedTable = {0}, onBaseTable = {1}, partitionedTable = {2}") - public static Object[][] parameters() { - return new Object[][] { - {true, true, true}, - {true, true, false}, - {true, false, true}, - {true, false, false}, - {false, true, true}, - {false, true, false} - }; + getAmsHandler().getTableCommitMetas().remove(getOperationTable().id().buildTableIdentifier()); } private UnkeyedTable getOperationTable() { @@ -113,19 +113,17 @@ private org.apache.iceberg.DataFile getDataFile(int number) { } } - @Before - public void clearCommitMeta() { - getAmsHandler().getTableCommitMetas().remove(getOperationTable().id().buildTableIdentifier()); - } - - @Test - public void testTraceAppendFiles() { + @ParameterizedTest(name = "keyedTable = {0}, onBaseTable = {1}, partitionedTable = {2}") + @MethodSource("parameters") + public void testTraceAppendFiles( + boolean keyedTable, boolean onBaseTable, boolean partitionedTable) throws IOException { + prepare(keyedTable, onBaseTable, partitionedTable); UnkeyedTable operationTable = getOperationTable(); operationTable.newAppend().appendFile(getDataFile(1)).appendFile(getDataFile(2)).commit(); List tableCommitMetas = getAmsHandler().getTableCommitMetas().get(operationTable.id().buildTableIdentifier()); - Assert.assertEquals(0, tableCommitMetas.size()); + Assertions.assertEquals(0, tableCommitMetas.size()); TableCommitMeta commitMeta = tableCommitMetas.get(0); validateCommitMeta( commitMeta, @@ -134,13 +132,16 @@ public void testTraceAppendFiles() { new org.apache.iceberg.DataFile[] {}); } - @Test - public void testTraceAppendFilesInTx() { + @ParameterizedTest(name = "keyedTable = {0}, onBaseTable = {1}, partitionedTable = {2}") + @MethodSource("parameters") + public void testTraceAppendFilesInTx( + boolean keyedTable, boolean onBaseTable, boolean partitionedTable) throws IOException { + prepare(keyedTable, onBaseTable, partitionedTable); UnkeyedTable operationTable = getOperationTable(); Transaction transaction = operationTable.newTransaction(); transaction.newAppend().appendFile(getDataFile(1)).appendFile(getDataFile(2)).commit(); - Assert.assertFalse( + Assertions.assertFalse( getAmsHandler() .getTableCommitMetas() .containsKey(operationTable.id().buildTableIdentifier())); @@ -149,7 +150,7 @@ public void testTraceAppendFilesInTx() { List tableCommitMetas = getAmsHandler().getTableCommitMetas().get(operationTable.id().buildTableIdentifier()); - Assert.assertEquals(1, tableCommitMetas.size()); + Assertions.assertEquals(1, tableCommitMetas.size()); TableCommitMeta commitMeta = tableCommitMetas.get(0); validateCommitMeta( commitMeta, @@ -158,8 +159,11 @@ public void testTraceAppendFilesInTx() { new org.apache.iceberg.DataFile[] {}); } - @Test - public void testTraceAppendFilesByOptimize() { + @ParameterizedTest(name = "keyedTable = {0}, onBaseTable = {1}, partitionedTable = {2}") + @MethodSource("parameters") + public void testTraceAppendFilesByOptimize( + boolean keyedTable, boolean onBaseTable, boolean partitionedTable) throws IOException { + prepare(keyedTable, onBaseTable, partitionedTable); UnkeyedTable operationTable = getOperationTable(); operationTable .newAppend() @@ -170,9 +174,9 @@ public void testTraceAppendFilesByOptimize() { List tableCommitMetas = getAmsHandler().getTableCommitMetas().get(operationTable.id().buildTableIdentifier()); - Assert.assertEquals(1, tableCommitMetas.size()); + Assertions.assertEquals(1, tableCommitMetas.size()); TableCommitMeta commitMeta = tableCommitMetas.get(0); - Assert.assertSame(commitMeta.getCommitMetaProducer(), CommitMetaProducer.OPTIMIZE); + Assertions.assertSame(commitMeta.getCommitMetaProducer(), CommitMetaProducer.OPTIMIZE); validateCommitMeta( commitMeta, DataOperations.APPEND, @@ -180,14 +184,17 @@ public void testTraceAppendFilesByOptimize() { new org.apache.iceberg.DataFile[] {}); } - @Test - public void testTraceFastAppend() { + @ParameterizedTest(name = "keyedTable = {0}, onBaseTable = {1}, partitionedTable = {2}") + @MethodSource("parameters") + public void testTraceFastAppend(boolean keyedTable, boolean onBaseTable, boolean partitionedTable) + throws IOException { + prepare(keyedTable, onBaseTable, partitionedTable); UnkeyedTable operationTable = getOperationTable(); operationTable.newFastAppend().appendFile(getDataFile(1)).appendFile(getDataFile(2)).commit(); List tableCommitMetas = getAmsHandler().getTableCommitMetas().get(operationTable.id().buildTableIdentifier()); - Assert.assertEquals(1, tableCommitMetas.size()); + Assertions.assertEquals(1, tableCommitMetas.size()); TableCommitMeta commitMeta = tableCommitMetas.get(0); validateCommitMeta( commitMeta, @@ -196,50 +203,56 @@ public void testTraceFastAppend() { new org.apache.iceberg.DataFile[] {}); } - @Test - public void testTraceAppendNoneFiles() { + @ParameterizedTest(name = "keyedTable = {0}, onBaseTable = {1}, partitionedTable = {2}") + @MethodSource("parameters") + public void testTraceAppendNoneFiles( + boolean keyedTable, boolean onBaseTable, boolean partitionedTable) throws IOException { + prepare(keyedTable, onBaseTable, partitionedTable); UnkeyedTable operationTable = getOperationTable(); operationTable.newAppend().commit(); List appendTableCommitMetas = getAmsHandler().getTableCommitMetas().get(operationTable.id().buildTableIdentifier()); - Assert.assertEquals(1, appendTableCommitMetas.size()); - Assert.assertNotNull(appendTableCommitMetas.get(0).getChanges()); + Assertions.assertEquals(1, appendTableCommitMetas.size()); + Assertions.assertNotNull(appendTableCommitMetas.get(0).getChanges()); Transaction overwriteTransaction = operationTable.newTransaction(); overwriteTransaction.newOverwrite().commit(); overwriteTransaction.commitTransaction(); List overwriteTableCommitMetas = getAmsHandler().getTableCommitMetas().get(operationTable.id().buildTableIdentifier()); - Assert.assertEquals(2, overwriteTableCommitMetas.size()); - Assert.assertEquals(1, overwriteTableCommitMetas.get(1).getChanges().size()); + Assertions.assertEquals(2, overwriteTableCommitMetas.size()); + Assertions.assertEquals(1, overwriteTableCommitMetas.get(1).getChanges().size()); Transaction rewriteTransaction = operationTable.newTransaction(); rewriteTransaction.newRewrite().commit(); rewriteTransaction.commitTransaction(); List rewriteTableCommitMetas = getAmsHandler().getTableCommitMetas().get(operationTable.id().buildTableIdentifier()); - Assert.assertEquals(3, rewriteTableCommitMetas.size()); - Assert.assertEquals(1, rewriteTableCommitMetas.get(2).getChanges().size()); + Assertions.assertEquals(3, rewriteTableCommitMetas.size()); + Assertions.assertEquals(1, rewriteTableCommitMetas.get(2).getChanges().size()); getMixedTable().updateSchema().commit(); List updateSchemaCommitMetas = getAmsHandler().getTableCommitMetas().get(operationTable.id().buildTableIdentifier()); - Assert.assertEquals(3, updateSchemaCommitMetas.size()); + Assertions.assertEquals(3, updateSchemaCommitMetas.size()); getMixedTable().updateProperties().commit(); List updatePropertiesCommitMetas = getAmsHandler().getTableCommitMetas().get(operationTable.id().buildTableIdentifier()); - Assert.assertEquals(4, updatePropertiesCommitMetas.size()); - Assert.assertNull(rewriteTableCommitMetas.get(3).getChanges()); + Assertions.assertEquals(4, updatePropertiesCommitMetas.size()); + Assertions.assertNull(rewriteTableCommitMetas.get(3).getChanges()); } - @Test - public void testTraceFastAppendInTx() { + @ParameterizedTest(name = "keyedTable = {0}, onBaseTable = {1}, partitionedTable = {2}") + @MethodSource("parameters") + public void testTraceFastAppendInTx( + boolean keyedTable, boolean onBaseTable, boolean partitionedTable) throws IOException { + prepare(keyedTable, onBaseTable, partitionedTable); UnkeyedTable operationTable = getOperationTable(); Transaction transaction = operationTable.newTransaction(); transaction.newFastAppend().appendFile(getDataFile(1)).appendFile(getDataFile(2)).commit(); - Assert.assertFalse( + Assertions.assertFalse( getAmsHandler() .getTableCommitMetas() .containsKey(operationTable.id().buildTableIdentifier())); @@ -247,7 +260,7 @@ public void testTraceFastAppendInTx() { transaction.commitTransaction(); List tableCommitMetas = getAmsHandler().getTableCommitMetas().get(operationTable.id().buildTableIdentifier()); - Assert.assertEquals(1, tableCommitMetas.size()); + Assertions.assertEquals(1, tableCommitMetas.size()); TableCommitMeta commitMeta = tableCommitMetas.get(0); validateCommitMeta( commitMeta, @@ -256,8 +269,11 @@ public void testTraceFastAppendInTx() { new org.apache.iceberg.DataFile[] {}); } - @Test - public void testTraceFastAppendByOptimize() { + @ParameterizedTest(name = "keyedTable = {0}, onBaseTable = {1}, partitionedTable = {2}") + @MethodSource("parameters") + public void testTraceFastAppendByOptimize( + boolean keyedTable, boolean onBaseTable, boolean partitionedTable) throws IOException { + prepare(keyedTable, onBaseTable, partitionedTable); UnkeyedTable operationTable = getOperationTable(); operationTable .newFastAppend() @@ -268,9 +284,9 @@ public void testTraceFastAppendByOptimize() { List tableCommitMetas = getAmsHandler().getTableCommitMetas().get(operationTable.id().buildTableIdentifier()); - Assert.assertEquals(1, tableCommitMetas.size()); + Assertions.assertEquals(1, tableCommitMetas.size()); TableCommitMeta commitMeta = tableCommitMetas.get(0); - Assert.assertSame(commitMeta.getCommitMetaProducer(), CommitMetaProducer.OPTIMIZE); + Assertions.assertSame(commitMeta.getCommitMetaProducer(), CommitMetaProducer.OPTIMIZE); validateCommitMeta( commitMeta, DataOperations.APPEND, @@ -278,8 +294,11 @@ public void testTraceFastAppendByOptimize() { new org.apache.iceberg.DataFile[] {}); } - @Test - public void testTraceOverwrite() { + @ParameterizedTest(name = "keyedTable = {0}, onBaseTable = {1}, partitionedTable = {2}") + @MethodSource("parameters") + public void testTraceOverwrite(boolean keyedTable, boolean onBaseTable, boolean partitionedTable) + throws IOException { + prepare(keyedTable, onBaseTable, partitionedTable); UnkeyedTable operationTable = getOperationTable(); operationTable.newFastAppend().appendFile(getDataFile(1)).appendFile(getDataFile(2)).commit(); @@ -292,7 +311,7 @@ public void testTraceOverwrite() { List tableCommitMetas = getAmsHandler().getTableCommitMetas().get(operationTable.id().buildTableIdentifier()); - Assert.assertEquals(2, tableCommitMetas.size()); + Assertions.assertEquals(2, tableCommitMetas.size()); TableCommitMeta commitMeta = tableCommitMetas.get(1); validateCommitMeta( commitMeta, @@ -301,8 +320,11 @@ public void testTraceOverwrite() { new org.apache.iceberg.DataFile[] {getDataFile(1), getDataFile(2)}); } - @Test - public void testTraceOverwriteInTx() { + @ParameterizedTest(name = "keyedTable = {0}, onBaseTable = {1}, partitionedTable = {2}") + @MethodSource("parameters") + public void testTraceOverwriteInTx( + boolean keyedTable, boolean onBaseTable, boolean partitionedTable) throws IOException { + prepare(keyedTable, onBaseTable, partitionedTable); UnkeyedTable operationTable = getOperationTable(); operationTable.newFastAppend().appendFile(getDataFile(1)).appendFile(getDataFile(2)).commit(); @@ -314,7 +336,7 @@ public void testTraceOverwriteInTx() { .addFile(getDataFile(3)) .commit(); - Assert.assertEquals( + Assertions.assertEquals( 1, getAmsHandler() .getTableCommitMetas() @@ -324,7 +346,7 @@ public void testTraceOverwriteInTx() { transaction.commitTransaction(); List tableCommitMetas = getAmsHandler().getTableCommitMetas().get(operationTable.id().buildTableIdentifier()); - Assert.assertEquals(2, tableCommitMetas.size()); + Assertions.assertEquals(2, tableCommitMetas.size()); TableCommitMeta commitMeta = tableCommitMetas.get(1); validateCommitMeta( commitMeta, @@ -333,8 +355,11 @@ public void testTraceOverwriteInTx() { new org.apache.iceberg.DataFile[] {getDataFile(1), getDataFile(2)}); } - @Test - public void testTraceOverwriteByOptimize() { + @ParameterizedTest(name = "keyedTable = {0}, onBaseTable = {1}, partitionedTable = {2}") + @MethodSource("parameters") + public void testTraceOverwriteByOptimize( + boolean keyedTable, boolean onBaseTable, boolean partitionedTable) throws IOException { + prepare(keyedTable, onBaseTable, partitionedTable); UnkeyedTable operationTable = getOperationTable(); operationTable.newFastAppend().appendFile(getDataFile(1)).appendFile(getDataFile(2)).commit(); @@ -348,9 +373,9 @@ public void testTraceOverwriteByOptimize() { List tableCommitMetas = getAmsHandler().getTableCommitMetas().get(operationTable.id().buildTableIdentifier()); - Assert.assertEquals(2, tableCommitMetas.size()); + Assertions.assertEquals(2, tableCommitMetas.size()); TableCommitMeta commitMeta = tableCommitMetas.get(1); - Assert.assertSame(commitMeta.getCommitMetaProducer(), CommitMetaProducer.OPTIMIZE); + Assertions.assertSame(commitMeta.getCommitMetaProducer(), CommitMetaProducer.OPTIMIZE); validateCommitMeta( commitMeta, DataOperations.OVERWRITE, @@ -358,8 +383,11 @@ public void testTraceOverwriteByOptimize() { new org.apache.iceberg.DataFile[] {getDataFile(1), getDataFile(2)}); } - @Test - public void testTraceRewrite() { + @ParameterizedTest(name = "keyedTable = {0}, onBaseTable = {1}, partitionedTable = {2}") + @MethodSource("parameters") + public void testTraceRewrite(boolean keyedTable, boolean onBaseTable, boolean partitionedTable) + throws IOException { + prepare(keyedTable, onBaseTable, partitionedTable); UnkeyedTable operationTable = getOperationTable(); operationTable.newFastAppend().appendFile(getDataFile(1)).appendFile(getDataFile(2)).commit(); @@ -371,7 +399,7 @@ public void testTraceRewrite() { List tableCommitMetas = getAmsHandler().getTableCommitMetas().get(operationTable.id().buildTableIdentifier()); - Assert.assertEquals(2, tableCommitMetas.size()); + Assertions.assertEquals(2, tableCommitMetas.size()); TableCommitMeta commitMeta = tableCommitMetas.get(1); validateCommitMeta( commitMeta, @@ -380,8 +408,11 @@ public void testTraceRewrite() { new org.apache.iceberg.DataFile[] {getDataFile(1), getDataFile(2)}); } - @Test - public void testTraceRewriteInTx() { + @ParameterizedTest(name = "keyedTable = {0}, onBaseTable = {1}, partitionedTable = {2}") + @MethodSource("parameters") + public void testTraceRewriteInTx( + boolean keyedTable, boolean onBaseTable, boolean partitionedTable) throws IOException { + prepare(keyedTable, onBaseTable, partitionedTable); UnkeyedTable operationTable = getOperationTable(); operationTable.newFastAppend().appendFile(getDataFile(1)).appendFile(getDataFile(2)).commit(); @@ -392,7 +423,7 @@ public void testTraceRewriteInTx() { Sets.newHashSet(getDataFile(1), getDataFile(2)), Sets.newHashSet(getDataFile(3))) .commit(); - Assert.assertEquals( + Assertions.assertEquals( 1, getAmsHandler() .getTableCommitMetas() @@ -402,7 +433,7 @@ public void testTraceRewriteInTx() { transaction.commitTransaction(); List tableCommitMetas = getAmsHandler().getTableCommitMetas().get(operationTable.id().buildTableIdentifier()); - Assert.assertEquals(2, tableCommitMetas.size()); + Assertions.assertEquals(2, tableCommitMetas.size()); TableCommitMeta commitMeta = tableCommitMetas.get(1); validateCommitMeta( commitMeta, @@ -411,8 +442,11 @@ public void testTraceRewriteInTx() { new org.apache.iceberg.DataFile[] {getDataFile(1), getDataFile(2)}); } - @Test - public void testTraceRewriteByOptimize() { + @ParameterizedTest(name = "keyedTable = {0}, onBaseTable = {1}, partitionedTable = {2}") + @MethodSource("parameters") + public void testTraceRewriteByOptimize( + boolean keyedTable, boolean onBaseTable, boolean partitionedTable) throws IOException { + prepare(keyedTable, onBaseTable, partitionedTable); UnkeyedTable operationTable = getOperationTable(); operationTable.newFastAppend().appendFile(getDataFile(1)).appendFile(getDataFile(2)).commit(); @@ -425,9 +459,9 @@ public void testTraceRewriteByOptimize() { List tableCommitMetas = getAmsHandler().getTableCommitMetas().get(operationTable.id().buildTableIdentifier()); - Assert.assertEquals(2, tableCommitMetas.size()); + Assertions.assertEquals(2, tableCommitMetas.size()); TableCommitMeta commitMeta = tableCommitMetas.get(1); - Assert.assertSame(commitMeta.getCommitMetaProducer(), CommitMetaProducer.OPTIMIZE); + Assertions.assertSame(commitMeta.getCommitMetaProducer(), CommitMetaProducer.OPTIMIZE); validateCommitMeta( commitMeta, DataOperations.REPLACE, @@ -435,15 +469,18 @@ public void testTraceRewriteByOptimize() { new org.apache.iceberg.DataFile[] {getDataFile(1), getDataFile(2)}); } - @Test - public void testMultipleOperationInTx() { + @ParameterizedTest(name = "keyedTable = {0}, onBaseTable = {1}, partitionedTable = {2}") + @MethodSource("parameters") + public void testMultipleOperationInTx( + boolean keyedTable, boolean onBaseTable, boolean partitionedTable) throws IOException { + prepare(keyedTable, onBaseTable, partitionedTable); UnkeyedTable operationTable = getOperationTable(); Transaction transaction = operationTable.newTransaction(); transaction.newAppend().appendFile(getDataFile(1)).appendFile(getDataFile(2)).commit(); transaction.newOverwrite().deleteFile(getDataFile(1)).addFile(getDataFile(3)).commit(); - Assert.assertFalse( + Assertions.assertFalse( getAmsHandler() .getTableCommitMetas() .containsKey(operationTable.id().buildTableIdentifier())); @@ -451,12 +488,12 @@ public void testMultipleOperationInTx() { transaction.commitTransaction(); List snapshots = Lists.newArrayList(operationTable.snapshots()); - Assert.assertEquals(2, snapshots.size()); + Assertions.assertEquals(2, snapshots.size()); List tableCommitMetas = getAmsHandler().getTableCommitMetas().get(operationTable.id().buildTableIdentifier()); - Assert.assertEquals(1, tableCommitMetas.size()); + Assertions.assertEquals(1, tableCommitMetas.size()); TableCommitMeta commitMeta = tableCommitMetas.get(0); - Assert.assertEquals(2, commitMeta.getChanges().size()); + Assertions.assertEquals(2, commitMeta.getChanges().size()); validateTableChange( snapshots.get(0), commitMeta.getChanges().get(0), @@ -469,8 +506,11 @@ public void testMultipleOperationInTx() { new org.apache.iceberg.DataFile[] {getDataFile(1)}); } - @Test - public void testMultipleOperationInTxByOptimize() { + @ParameterizedTest(name = "keyedTable = {0}, onBaseTable = {1}, partitionedTable = {2}") + @MethodSource("parameters") + public void testMultipleOperationInTxByOptimize( + boolean keyedTable, boolean onBaseTable, boolean partitionedTable) throws IOException { + prepare(keyedTable, onBaseTable, partitionedTable); UnkeyedTable operationTable = getOperationTable(); Transaction transaction = operationTable.newTransaction(); transaction @@ -487,7 +527,7 @@ public void testMultipleOperationInTxByOptimize() { .set(SnapshotSummary.SNAPSHOT_PRODUCER, CommitMetaProducer.OPTIMIZE.name()) .commit(); - Assert.assertFalse( + Assertions.assertFalse( getAmsHandler() .getTableCommitMetas() .containsKey(operationTable.id().buildTableIdentifier())); @@ -495,13 +535,13 @@ public void testMultipleOperationInTxByOptimize() { transaction.commitTransaction(); List snapshots = Lists.newArrayList(operationTable.snapshots()); - Assert.assertEquals(2, snapshots.size()); + Assertions.assertEquals(2, snapshots.size()); List tableCommitMetas = getAmsHandler().getTableCommitMetas().get(operationTable.id().buildTableIdentifier()); - Assert.assertEquals(1, tableCommitMetas.size()); + Assertions.assertEquals(1, tableCommitMetas.size()); TableCommitMeta commitMeta = tableCommitMetas.get(0); - Assert.assertSame(commitMeta.getCommitMetaProducer(), CommitMetaProducer.OPTIMIZE); - Assert.assertEquals(2, commitMeta.getChanges().size()); + Assertions.assertSame(commitMeta.getCommitMetaProducer(), CommitMetaProducer.OPTIMIZE); + Assertions.assertEquals(2, commitMeta.getChanges().size()); validateTableChange( snapshots.get(0), commitMeta.getChanges().get(0), @@ -514,9 +554,12 @@ public void testMultipleOperationInTxByOptimize() { new org.apache.iceberg.DataFile[] {getDataFile(1)}); } - @Test - public void testTracedReplacePartitions() { - Assume.assumeTrue(isPartitionedTable()); + @ParameterizedTest(name = "keyedTable = {0}, onBaseTable = {1}, partitionedTable = {2}") + @MethodSource("parameters") + public void testTracedReplacePartitions( + boolean keyedTable, boolean onBaseTable, boolean partitionedTable) throws IOException { + prepare(keyedTable, onBaseTable, partitionedTable); + Assumptions.assumeTrue(isPartitionedTable()); UnkeyedTable operationTable = getOperationTable(); operationTable .newFastAppend() @@ -532,7 +575,7 @@ public void testTracedReplacePartitions() { List tableCommitMetas = getAmsHandler().getTableCommitMetas().get(operationTable.id().buildTableIdentifier()); - Assert.assertEquals(2, tableCommitMetas.size()); + Assertions.assertEquals(2, tableCommitMetas.size()); TableCommitMeta commitMeta = tableCommitMetas.get(1); validateCommitMeta( commitMeta, @@ -545,9 +588,12 @@ public void testTracedReplacePartitions() { }); } - @Test - public void testTraceRemovePosDeleteInternal() throws Exception { - Assume.assumeTrue(isKeyedTable() && onBaseTable); + @ParameterizedTest(name = "keyedTable = {0}, onBaseTable = {1}, partitionedTable = {2}") + @MethodSource("parameters") + public void testTraceRemovePosDeleteInternal( + boolean keyedTable, boolean onBaseTable, boolean partitionedTable) throws Exception { + prepare(keyedTable, onBaseTable, partitionedTable); + Assumptions.assumeTrue(isKeyedTable() && this.onBaseTable); getMixedTable().asKeyedTable().baseTable().newAppend().appendFile(getDataFile(1)).commit(); SortedPosDeleteWriter writer = @@ -570,16 +616,19 @@ public void testTraceRemovePosDeleteInternal() throws Exception { List tableCommitMetas = getAmsHandler().getTableCommitMetas().get(getMixedTable().id().buildTableIdentifier()); - Assert.assertEquals(4, tableCommitMetas.size()); + Assertions.assertEquals(4, tableCommitMetas.size()); TableCommitMeta commitMeta = tableCommitMetas.get(tableCommitMetas.size() - 1); - Assert.assertEquals(1, commitMeta.getChanges().size()); + Assertions.assertEquals(1, commitMeta.getChanges().size()); TableChange tableChange = commitMeta.getChanges().get(0); - Assert.assertEquals(2, tableChange.deleteFiles.size()); + Assertions.assertEquals(2, tableChange.deleteFiles.size()); } - @Test - public void testTraceRemovePosDeleteInternalInTransaction() throws Exception { - Assume.assumeTrue(isKeyedTable() && onBaseTable); + @ParameterizedTest(name = "keyedTable = {0}, onBaseTable = {1}, partitionedTable = {2}") + @MethodSource("parameters") + public void testTraceRemovePosDeleteInternalInTransaction( + boolean keyedTable, boolean onBaseTable, boolean partitionedTable) throws Exception { + prepare(keyedTable, onBaseTable, partitionedTable); + Assumptions.assumeTrue(isKeyedTable() && this.onBaseTable); getMixedTable().asKeyedTable().baseTable().newAppend().appendFile(getDataFile(1)).commit(); SortedPosDeleteWriter writer = @@ -607,11 +656,11 @@ public void testTraceRemovePosDeleteInternalInTransaction() throws Exception { List tableCommitMetas = getAmsHandler().getTableCommitMetas().get(getMixedTable().id().buildTableIdentifier()); - Assert.assertEquals(4, tableCommitMetas.size()); + Assertions.assertEquals(4, tableCommitMetas.size()); TableCommitMeta commitMeta = tableCommitMetas.get(tableCommitMetas.size() - 1); - Assert.assertEquals(2, commitMeta.getChanges().size()); + Assertions.assertEquals(2, commitMeta.getChanges().size()); TableChange tableChange = commitMeta.getChanges().get(0); - Assert.assertEquals(2, tableChange.deleteFiles.size()); + Assertions.assertEquals(2, tableChange.deleteFiles.size()); } private String getExpectedInnerTable() { @@ -627,9 +676,9 @@ private void validateTableChange( TableChange tableChange, org.apache.iceberg.DataFile[] addFiles, org.apache.iceberg.DataFile[] deleteFiles) { - Assert.assertEquals(getExpectedInnerTable(), tableChange.getInnerTable()); - Assert.assertEquals(snapshot.snapshotId(), tableChange.getSnapshotId()); - Assert.assertEquals( + Assertions.assertEquals(getExpectedInnerTable(), tableChange.getInnerTable()); + Assertions.assertEquals(snapshot.snapshotId(), tableChange.getSnapshotId()); + Assertions.assertEquals( snapshot.parentId() == null ? -1 : getOperationTable().currentSnapshot().parentId(), tableChange.getParentSnapshotId()); @@ -643,16 +692,16 @@ private void validateCommitMeta( org.apache.iceberg.DataFile[] addFiles, org.apache.iceberg.DataFile[] deleteFiles) { - Assert.assertEquals( + Assertions.assertEquals( getMixedTable().id().buildTableIdentifier(), commitMeta.getTableIdentifier()); - Assert.assertEquals(operation, commitMeta.getAction()); - Assert.assertEquals(1, commitMeta.getChanges().size()); + Assertions.assertEquals(operation, commitMeta.getAction()); + Assertions.assertEquals(1, commitMeta.getChanges().size()); TableChange tableChange = commitMeta.getChanges().get(0); - Assert.assertEquals(getExpectedInnerTable(), tableChange.getInnerTable()); - Assert.assertEquals( + Assertions.assertEquals(getExpectedInnerTable(), tableChange.getInnerTable()); + Assertions.assertEquals( getOperationTable().currentSnapshot().snapshotId(), tableChange.getSnapshotId()); - Assert.assertEquals( + Assertions.assertEquals( getOperationTable().currentSnapshot().parentId() == null ? -1 : getOperationTable().currentSnapshot().parentId(), @@ -664,26 +713,26 @@ private void validateCommitMeta( private void validateDataFile( List dataFiles, org.apache.iceberg.DataFile... icebergFiles) { - Assert.assertEquals(icebergFiles.length, dataFiles.size()); + Assertions.assertEquals(icebergFiles.length, dataFiles.size()); Map icebergFilesMap = Maps.newHashMap(); Arrays.stream(icebergFiles).forEach(f -> icebergFilesMap.put(f.path().toString(), f)); for (DataFile validateFile : dataFiles) { org.apache.iceberg.DataFile icebergFile = icebergFilesMap.get(validateFile.getPath()); - Assert.assertEquals(icebergFile.path(), validateFile.getPath()); - Assert.assertEquals(icebergFile.fileSizeInBytes(), validateFile.getFileSize()); - Assert.assertEquals(icebergFile.recordCount(), validateFile.getRecordCount()); - Assert.assertEquals( + Assertions.assertEquals(icebergFile.path(), validateFile.getPath()); + Assertions.assertEquals(icebergFile.fileSizeInBytes(), validateFile.getFileSize()); + Assertions.assertEquals(icebergFile.recordCount(), validateFile.getRecordCount()); + Assertions.assertEquals( onBaseTable ? DataFileType.BASE_FILE.name() : DataFileType.INSERT_FILE.name(), validateFile.getFileType()); - Assert.assertEquals(0, validateFile.getIndex()); - Assert.assertEquals(0, validateFile.getMask()); + Assertions.assertEquals(0, validateFile.getIndex()); + Assertions.assertEquals(0, validateFile.getMask()); if (isPartitionedTable()) { - Assert.assertEquals(getMixedTable().spec().specId(), validateFile.getSpecId()); - Assert.assertEquals(1, validateFile.getPartitionSize()); - Assert.assertEquals( + Assertions.assertEquals(getMixedTable().spec().specId(), validateFile.getSpecId()); + Assertions.assertEquals(1, validateFile.getPartitionSize()); + Assertions.assertEquals( getMixedTable().spec().fields().get(0).name(), validateFile.getPartition().get(0).getName()); - Assert.assertEquals( + Assertions.assertEquals( getMixedTable().spec().partitionToPath(icebergFile.partition()), getMixedTable().spec().fields().get(0).name() + "=" diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/trace/TestTableWatermark.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/trace/TestTableWatermark.java index 9ac3bfa057..6ac5691706 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/trace/TestTableWatermark.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/trace/TestTableWatermark.java @@ -33,34 +33,35 @@ import org.apache.iceberg.Transaction; import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Types; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.Map; import java.util.function.Function; +import java.util.stream.Stream; -@RunWith(Parameterized.class) public class TestTableWatermark extends TableTestBase { - private final boolean onBaseTable; + private boolean onBaseTable; private UnkeyedTable operationTable; - public TestTableWatermark(boolean keyedTable, boolean onBaseTable) { - super( + public static Stream parameters() { + return Stream.of( + Arguments.of(true, true), Arguments.of(true, false), Arguments.of(false, true)); + } + + private void prepare(boolean keyedTable, boolean onBaseTable) throws IOException { + setupTable( new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), new BasicTableTestHelper(keyedTable, true)); this.onBaseTable = onBaseTable; } - @Parameterized.Parameters(name = "keyedTable = {0}, onBaseTable = {1}") - public static Object[][] parameters() { - return new Object[][] {{true, true}, {true, false}, {false, true}}; - } - private UnkeyedTable getOperationTable() { if (operationTable == null) { MixedTable mixedTable = getMixedTable(); @@ -81,8 +82,11 @@ private UnkeyedTable getOperationTable() { return operationTable; } - @Test - public void testChangeWatermarkWithAppendFiles() { + @ParameterizedTest(name = "keyedTable = {0}, onBaseTable = {1}") + @MethodSource("parameters") + public void testChangeWatermarkWithAppendFiles(boolean keyedTable, boolean onBaseTable) + throws IOException { + prepare(keyedTable, onBaseTable); testTableWatermark( addFile -> { getOperationTable().newAppend().appendFile(addFile).commit(); @@ -90,8 +94,11 @@ public void testChangeWatermarkWithAppendFiles() { }); } - @Test - public void testChangeWatermarkWithAppendFilesInTx() { + @ParameterizedTest(name = "keyedTable = {0}, onBaseTable = {1}") + @MethodSource("parameters") + public void testChangeWatermarkWithAppendFilesInTx(boolean keyedTable, boolean onBaseTable) + throws IOException { + prepare(keyedTable, onBaseTable); testTableWatermark( addFile -> { Transaction transaction = getOperationTable().newTransaction(); @@ -101,8 +108,11 @@ public void testChangeWatermarkWithAppendFilesInTx() { }); } - @Test - public void testChangeWatermarkWithOverwriteFiles() { + @ParameterizedTest(name = "keyedTable = {0}, onBaseTable = {1}") + @MethodSource("parameters") + public void testChangeWatermarkWithOverwriteFiles(boolean keyedTable, boolean onBaseTable) + throws IOException { + prepare(keyedTable, onBaseTable); testTableWatermark( addFile -> { getOperationTable().newOverwrite().addFile(addFile).commit(); @@ -110,8 +120,11 @@ public void testChangeWatermarkWithOverwriteFiles() { }); } - @Test - public void testChangeWatermarkWithOverwriteFilesInTx() { + @ParameterizedTest(name = "keyedTable = {0}, onBaseTable = {1}") + @MethodSource("parameters") + public void testChangeWatermarkWithOverwriteFilesInTx(boolean keyedTable, boolean onBaseTable) + throws IOException { + prepare(keyedTable, onBaseTable); testTableWatermark( addFile -> { Transaction transaction = getOperationTable().newTransaction(); @@ -121,8 +134,11 @@ public void testChangeWatermarkWithOverwriteFilesInTx() { }); } - @Test - public void testChangeWatermarkWithReplacePartitions() { + @ParameterizedTest(name = "keyedTable = {0}, onBaseTable = {1}") + @MethodSource("parameters") + public void testChangeWatermarkWithReplacePartitions(boolean keyedTable, boolean onBaseTable) + throws IOException { + prepare(keyedTable, onBaseTable); testTableWatermark( addFile -> { getOperationTable().newReplacePartitions().addFile(addFile).commit(); @@ -130,8 +146,11 @@ public void testChangeWatermarkWithReplacePartitions() { }); } - @Test - public void testChangeWatermarkWithReplacePartitionsInTx() { + @ParameterizedTest(name = "keyedTable = {0}, onBaseTable = {1}") + @MethodSource("parameters") + public void testChangeWatermarkWithReplacePartitionsInTx(boolean keyedTable, boolean onBaseTable) + throws IOException { + prepare(keyedTable, onBaseTable); testTableWatermark( addFile -> { Transaction transaction = getOperationTable().newTransaction(); @@ -141,8 +160,11 @@ public void testChangeWatermarkWithReplacePartitionsInTx() { }); } - @Test - public void testChangeWatermarkWithRowDelta() { + @ParameterizedTest(name = "keyedTable = {0}, onBaseTable = {1}") + @MethodSource("parameters") + public void testChangeWatermarkWithRowDelta(boolean keyedTable, boolean onBaseTable) + throws IOException { + prepare(keyedTable, onBaseTable); testTableWatermark( addFile -> { getOperationTable().newRowDelta().addRows(addFile).commit(); @@ -150,8 +172,11 @@ public void testChangeWatermarkWithRowDelta() { }); } - @Test - public void testChangeWatermarkWithRowDeltaFilesInTx() { + @ParameterizedTest(name = "keyedTable = {0}, onBaseTable = {1}") + @MethodSource("parameters") + public void testChangeWatermarkWithRowDeltaFilesInTx(boolean keyedTable, boolean onBaseTable) + throws IOException { + prepare(keyedTable, onBaseTable); testTableWatermark( addFile -> { Transaction transaction = getOperationTable().newTransaction(); @@ -192,7 +217,7 @@ private void testTableWatermark(Function tableOperation) { .withMetrics(metrics) .build(); tableOperation.apply(file1); - Assert.assertEquals( + Assertions.assertEquals( start - 20000, TablePropertyUtil.getTableWatermark(getMixedTable().properties())); } } diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestCompatiblePropertyUtil.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestCompatiblePropertyUtil.java index 5bdf6dcd2e..40178dcec1 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestCompatiblePropertyUtil.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestCompatiblePropertyUtil.java @@ -20,8 +20,8 @@ import org.apache.amoro.shade.guava32.com.google.common.collect.Maps; import org.apache.amoro.table.TableProperties; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.util.Map; @@ -30,7 +30,7 @@ public class TestCompatiblePropertyUtil { @Test public void testGetNewProperty() { Map properties = Maps.newHashMap(); - Assert.assertEquals( + Assertions.assertEquals( TableProperties.ENABLE_SELF_OPTIMIZING_DEFAULT, CompatiblePropertyUtil.propertyAsBoolean( properties, @@ -38,7 +38,7 @@ public void testGetNewProperty() { TableProperties.ENABLE_SELF_OPTIMIZING_DEFAULT)); properties.put(TableProperties.ENABLE_SELF_OPTIMIZING, "false"); - Assert.assertFalse( + Assertions.assertFalse( CompatiblePropertyUtil.propertyAsBoolean( properties, TableProperties.ENABLE_SELF_OPTIMIZING, @@ -49,7 +49,7 @@ public void testGetNewProperty() { public void testGetLegacyProperty() { Map properties = Maps.newHashMap(); properties.put(TableProperties.ENABLE_SELF_OPTIMIZING, "false"); - Assert.assertFalse( + Assertions.assertFalse( CompatiblePropertyUtil.propertyAsBoolean( properties, TableProperties.ENABLE_SELF_OPTIMIZING, diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestExpressionUtil.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestExpressionUtil.java index 03707da7b7..fb282e9756 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestExpressionUtil.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestExpressionUtil.java @@ -24,8 +24,8 @@ import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.sql.Date; import java.sql.Time; @@ -122,6 +122,6 @@ public void testConvertSqlToIcebergExpressionByType(T exprValue, String sqlV } private void assertEqualExpressions(Expression exp1, Expression exp2) { - Assert.assertEquals(exp1.toString(), exp2.toString()); + Assertions.assertEquals(exp1.toString(), exp2.toString()); } } diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestFileUtil.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestFileUtil.java index a466e66120..6600a565db 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestFileUtil.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestFileUtil.java @@ -27,16 +27,17 @@ import org.apache.iceberg.io.FileInfo; import org.apache.iceberg.io.InputFile; import org.apache.iceberg.io.OutputFile; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; -import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; import java.net.URI; +import java.nio.file.Path; import java.util.Iterator; import java.util.Objects; import java.util.Set; @@ -51,7 +52,7 @@ public void getFileName() { TableFileUtil.getFileName( "hdfs://easyops-sloth/user/warehouse/animal_partition_two/base/" + "opt_mon=202109/opt_day=26/00000-0-3-1-37128f07-0845-43d8-905b-bd69b4ca351c-0000000001.parquet"); - Assert.assertEquals( + Assertions.assertEquals( "00000-0-3-1-37128f07-0845-43d8-905b-bd69b4ca351c-0000000001.parquet", fileName); } @@ -61,7 +62,7 @@ public void getFileDir() { TableFileUtil.getFileDir( "hdfs://easyops-sloth/user/warehouse/animal_partition_two/base/" + "opt_mon=202109/opt_day=26/00000-0-3-1-37128f07-0845-43d8-905b-bd69b4ca351c-0000000001.parquet"); - Assert.assertEquals( + Assertions.assertEquals( "hdfs://easyops-sloth/user/warehouse/animal_partition_two/base/opt_mon=202109/opt_day=26", fileDir); } @@ -77,10 +78,10 @@ public void getFileDir() { "a/b/c, a/b/c" }) public void testGetUriPath(String expected, String path) { - Assert.assertEquals(expected, TableFileUtil.getUriPath(path)); + Assertions.assertEquals(expected, TableFileUtil.getUriPath(path)); } - private static final TemporaryFolder temp = new TemporaryFolder(); + @TempDir public Path temp; static class LocalAuthenticatedFileIO implements AuthenticatedFileIO, SupportsFileSystemOperations { @@ -113,10 +114,9 @@ public void deleteFile(String path) { @Override public void makeDirectories(String path) { - try { - temp.newFolder(path); - } catch (IOException e) { - throw new RuntimeException(e); + File dir = new File(path); + if (!dir.exists() && !dir.mkdirs()) { + throw new UncheckedIOException(new IOException("Failed to create directory: " + path)); } } @@ -176,16 +176,15 @@ public void deletePrefix(String prefix) { @Test public void testDeleteEmptyDirectory() throws IOException { - temp.create(); - String dataLocation = temp.newFolder("data").getAbsolutePath(); - String metadataLocation = temp.newFolder("metadata").getAbsolutePath(); - temp.newFile("metadata/metadata.json"); + String dataLocation = newFolder(temp, "data").getAbsolutePath(); + String metadataLocation = newFolder(temp, "metadata").getAbsolutePath(); + newFile(temp, "metadata/metadata.json"); - File emptyPartition1 = temp.newFolder("data/partition1"); - File emptyPartition2 = temp.newFolder("data/partition2"); - File emptySubPartition = temp.newFolder("data/partition2/sub-partition1"); - File partition3 = temp.newFolder("data/partition3"); - File file = temp.newFile("data/partition3/data-file-1"); + File emptyPartition1 = newFolder(temp, "data/partition1"); + File emptyPartition2 = newFolder(temp, "data/partition2"); + File emptySubPartition = newFolder(temp, "data/partition2/sub-partition1"); + File partition3 = newFolder(temp, "data/partition3"); + File file = newFile(temp, "data/partition3/data-file-1"); Set exclude = Sets.newHashSet(file.getAbsolutePath()); try (LocalAuthenticatedFileIO io = new LocalAuthenticatedFileIO()) { @@ -196,10 +195,10 @@ public void testDeleteEmptyDirectory() throws IOException { partition3.getAbsolutePath(), metadataLocation) .forEach(f -> TableFileUtil.deleteEmptyDirectory(io, f, exclude)); - Assert.assertFalse(io.exists(emptyPartition1.getAbsolutePath())); - Assert.assertFalse(io.exists(emptyPartition2.getAbsolutePath())); - Assert.assertTrue(io.exists(dataLocation)); - Assert.assertTrue(io.exists(metadataLocation)); + Assertions.assertFalse(io.exists(emptyPartition1.getAbsolutePath())); + Assertions.assertFalse(io.exists(emptyPartition2.getAbsolutePath())); + Assertions.assertTrue(io.exists(dataLocation)); + Assertions.assertTrue(io.exists(metadataLocation)); } } @@ -209,7 +208,7 @@ public void testGetParent() { "hdfs://easyops-sloth/user/warehouse/animal_partition_two/base/" + "opt_mon=202109/opt_day=26/00000-0-3-1-37128f07-0845-43d8-905b-bd69b4ca351c-0000000001.parquet"; String parentDir = TableFileUtil.getParent(filePath); - Assert.assertEquals( + Assertions.assertEquals( "hdfs://easyops-sloth/user/warehouse/animal_partition_two/base/" + "opt_mon=202109/opt_day=26", parentDir); @@ -219,19 +218,19 @@ public void testGetParent() { "s3://my-bucket/user/warehouse/animal_partition_two/base/" + "opt_mon=202109/opt_day=26/00000-0-3-1-37128f07-0845-43d8-905b-bd69b4ca351c-0000000001.parquet"; parentDir = TableFileUtil.getParent(filePath); - Assert.assertEquals( + Assertions.assertEquals( "s3://my-bucket/user/warehouse/animal_partition_two/base/" + "opt_mon=202109/opt_day=26", parentDir); // test wrapped by URI URI uri = URI.create(filePath); parentDir = TableFileUtil.getParent(uri.toString()); - Assert.assertEquals( + Assertions.assertEquals( "s3://my-bucket/user/warehouse/animal_partition_two/base/" + "opt_mon=202109/opt_day=26", parentDir); // lose scheme when getting path from URI parentDir = TableFileUtil.getParent(uri.getPath()); - Assert.assertEquals( + Assertions.assertEquals( "/user/warehouse/animal_partition_two/base/opt_mon=202109/opt_day=26", parentDir); // test no scheme @@ -239,12 +238,32 @@ public void testGetParent() { "/user/warehouse/animal_partition_two/base/opt_mon=202109/opt_day=26/" + "00000-0-3-1-37128f07-0845-43d8-905b-bd69b4ca351c-0000000001.parquet"; parentDir = TableFileUtil.getParent(filePath); - Assert.assertEquals( + Assertions.assertEquals( "/user/warehouse/animal_partition_two/base/opt_mon=202109/opt_day=26", parentDir); // test root path filePath = "/00000-0-3-1-37128f07-0845-43d8-905b-bd69b4ca351c-0000000001.parquet"; parentDir = TableFileUtil.getParent(filePath); - Assert.assertEquals("/", parentDir); + Assertions.assertEquals("/", parentDir); + } + + private static File newFolder(Path root, String relative) throws IOException { + File dir = root.resolve(relative).toFile(); + if (!dir.exists() && !dir.mkdirs()) { + throw new IOException("Failed to create directory: " + dir); + } + return dir; + } + + private static File newFile(Path root, String relative) throws IOException { + File file = root.resolve(relative).toFile(); + File parent = file.getParentFile(); + if (parent != null && !parent.exists() && !parent.mkdirs()) { + throw new IOException("Failed to create directory: " + parent); + } + if (!file.createNewFile()) { + throw new IOException("Failed to create file: " + file); + } + return file; } } diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestKeyedExpressionUtil.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestKeyedExpressionUtil.java index 4b547f69e7..354a53da99 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestKeyedExpressionUtil.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestKeyedExpressionUtil.java @@ -34,26 +34,20 @@ import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.types.Types; -import org.junit.Assert; -import org.junit.Assume; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Set; +import java.util.stream.Stream; -@RunWith(Parameterized.class) public class TestKeyedExpressionUtil extends TableTestBase { - public TestKeyedExpressionUtil(PartitionSpec partitionSpec) { - super( - new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), - new BasicTableTestHelper(TABLE_SCHEMA, true, partitionSpec)); - } - public static final Schema TABLE_SCHEMA = new Schema( Types.NestedField.required(1, "id", Types.IntegerType.get()), @@ -61,23 +55,26 @@ public TestKeyedExpressionUtil(PartitionSpec partitionSpec) { Types.NestedField.required(3, "ts", Types.LongType.get()), Types.NestedField.optional(4, "op_time", Types.TimestampType.withoutZone())); - @Parameterized.Parameters(name = "{0}") - public static Object[] parameters() { - return new Object[][] { - {PartitionSpec.builderFor(TABLE_SCHEMA).identity("op_time").build()}, - {PartitionSpec.builderFor(TABLE_SCHEMA).bucket("name", 2).build()}, - {PartitionSpec.builderFor(TABLE_SCHEMA).truncate("ts", 10).build()}, - {PartitionSpec.builderFor(TABLE_SCHEMA).year("op_time").build()}, - {PartitionSpec.builderFor(TABLE_SCHEMA).month("op_time").build()}, - {PartitionSpec.builderFor(TABLE_SCHEMA).day("op_time").build()}, - {PartitionSpec.builderFor(TABLE_SCHEMA).hour("op_time").build()}, - {PartitionSpec.unpartitioned()} - }; + public static Stream parameters() { + return Stream.of( + Arguments.of(PartitionSpec.builderFor(TABLE_SCHEMA).identity("op_time").build()), + Arguments.of(PartitionSpec.builderFor(TABLE_SCHEMA).bucket("name", 2).build()), + Arguments.of(PartitionSpec.builderFor(TABLE_SCHEMA).truncate("ts", 10).build()), + Arguments.of(PartitionSpec.builderFor(TABLE_SCHEMA).year("op_time").build()), + Arguments.of(PartitionSpec.builderFor(TABLE_SCHEMA).month("op_time").build()), + Arguments.of(PartitionSpec.builderFor(TABLE_SCHEMA).day("op_time").build()), + Arguments.of(PartitionSpec.builderFor(TABLE_SCHEMA).hour("op_time").build()), + Arguments.of(PartitionSpec.unpartitioned())); } - @Test - public void testKeyedConvertPartitionStructLikeToDataFilter() { - Assume.assumeTrue(isKeyedTable()); + @ParameterizedTest(name = "{0}") + @MethodSource("parameters") + public void testKeyedConvertPartitionStructLikeToDataFilter(PartitionSpec partitionSpec) + throws IOException { + setupTable( + new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), + new BasicTableTestHelper(TABLE_SCHEMA, true, partitionSpec)); + Assumptions.assumeTrue(isKeyedTable()); ArrayList baseStoreRecords = Lists.newArrayList( // hash("111") = -210118348, hash("222") = -699778209 @@ -137,8 +134,8 @@ private void assertPlanHalfWithPartitionFilter(Expression partitionFilter) { throw new RuntimeException(e); } - Assert.assertEquals(4, baseDataFiles.size()); - Assert.assertEquals(4, insertFiles.size()); + Assertions.assertEquals(4, baseDataFiles.size()); + Assertions.assertEquals(4, insertFiles.size()); baseDataFiles.clear(); insertFiles.clear(); @@ -157,11 +154,11 @@ private void assertPlanHalfWithPartitionFilter(Expression partitionFilter) { throw new RuntimeException(e); } if (isPartitionedTable()) { - Assert.assertEquals(2, baseDataFiles.size()); - Assert.assertEquals(2, insertFiles.size()); + Assertions.assertEquals(2, baseDataFiles.size()); + Assertions.assertEquals(2, insertFiles.size()); } else { - Assert.assertEquals(4, baseDataFiles.size()); - Assert.assertEquals(4, insertFiles.size()); + Assertions.assertEquals(4, baseDataFiles.size()); + Assertions.assertEquals(4, insertFiles.size()); } } } diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestMixedDataFiles.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestMixedDataFiles.java index 1faef4be38..1ddea0ff20 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestMixedDataFiles.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestMixedDataFiles.java @@ -26,8 +26,8 @@ import org.apache.iceberg.data.InternalRecordWrapper; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.StructLikeWrapper; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.time.LocalDateTime; import java.time.OffsetDateTime; @@ -51,7 +51,7 @@ public void testMonthPartition() { p1.set(partitionKey); StructLikeWrapper p2 = StructLikeWrapper.forType(spec.partitionType()); p2.set(partitionData); - Assert.assertEquals(p1, p2); + Assertions.assertEquals(p1, p2); } @Test @@ -68,7 +68,7 @@ public void testDaysPartition() { p1.set(partitionKey); StructLikeWrapper p2 = StructLikeWrapper.forType(spec.partitionType()); p2.set(partitionData); - Assert.assertEquals(p1, p2); + Assertions.assertEquals(p1, p2); } @Test @@ -87,7 +87,7 @@ public void testHoursPartition() { p1.set(partitionKey); StructLikeWrapper p2 = StructLikeWrapper.forType(spec.partitionType()); p2.set(partitionData); - Assert.assertEquals(p1, p2); + Assertions.assertEquals(p1, p2); partitionKey.partition( internalRecordWrapper.wrap(record.copy("dt", LocalDateTime.parse("2022-11-11T12:00:00")))); @@ -95,7 +95,7 @@ public void testHoursPartition() { partitionData = MixedDataFiles.data(spec, partitionPath); p1.set(partitionKey); p2.set(partitionData); - Assert.assertEquals(p1, p2); + Assertions.assertEquals(p1, p2); partitionKey.partition( internalRecordWrapper.wrap(record.copy("dt", LocalDateTime.parse("2022-11-11T15:30:00")))); @@ -103,7 +103,7 @@ public void testHoursPartition() { partitionData = MixedDataFiles.data(spec, partitionPath); p1.set(partitionKey); p2.set(partitionData); - Assert.assertEquals(p1, p2); + Assertions.assertEquals(p1, p2); partitionKey.partition( internalRecordWrapper.wrap(record.copy("dt", LocalDateTime.parse("2022-11-12T00:00:00")))); @@ -111,7 +111,7 @@ public void testHoursPartition() { partitionData = MixedDataFiles.data(spec, partitionPath); p1.set(partitionKey); p2.set(partitionData); - Assert.assertEquals(p1, p2); + Assertions.assertEquals(p1, p2); } @Test @@ -128,7 +128,7 @@ public void testBucketPartition() { p1.set(partitionKey); StructLikeWrapper p2 = StructLikeWrapper.forType(spec.partitionType()); p2.set(partitionData); - Assert.assertEquals(p1, p2); + Assertions.assertEquals(p1, p2); } @Test @@ -145,7 +145,7 @@ public void testTruncatePartition() { p1.set(partitionKey); StructLikeWrapper p2 = StructLikeWrapper.forType(spec.partitionType()); p2.set(partitionData); - Assert.assertEquals(p1, p2); + Assertions.assertEquals(p1, p2); } @Test @@ -155,7 +155,7 @@ public void testNullPartition() { PartitionKey partitionKey = new PartitionKey(spec, schema); String partitionPath = "name=null"; StructLike partitionData = MixedDataFiles.data(spec, partitionPath); - Assert.assertNull(partitionData.get(0, Types.StringType.get().typeId().javaClass())); + Assertions.assertNull(partitionData.get(0, Types.StringType.get().typeId().javaClass())); } @Test @@ -173,14 +173,14 @@ public void testSpecialCharactersPartition() { record.copy("day", "2023-01-01").copy("name", "AAA BBB/CCC=_*-\\%"))); String partitionToPath = spec.partitionToPath(partitionKey); - Assert.assertEquals("day=2023-01-01/name=AAA+BBB%2FCCC%3D_*-%5C%25", partitionToPath); + Assertions.assertEquals("day=2023-01-01/name=AAA+BBB%2FCCC%3D_*-%5C%25", partitionToPath); GenericRecord partitionData = MixedDataFiles.data(spec, partitionToPath); StructLikeWrapper p1 = StructLikeWrapper.forType(spec.partitionType()); p1.set(partitionKey); StructLikeWrapper p2 = StructLikeWrapper.forType(spec.partitionType()); p2.set(partitionData); - Assert.assertEquals(p1, p2); + Assertions.assertEquals(p1, p2); } @Test @@ -199,7 +199,7 @@ public void testTimestampPartitionTZ() { p1.set(partitionKey); StructLikeWrapper p2 = StructLikeWrapper.forType(spec.partitionType()); p2.set(partitionData); - Assert.assertEquals(p1, p2); + Assertions.assertEquals(p1, p2); } @Test @@ -216,6 +216,6 @@ public void testTimestampPartitionNTZ() { p1.set(partitionKey); StructLikeWrapper p2 = StructLikeWrapper.forType(spec.partitionType()); p2.set(partitionData); - Assert.assertEquals(p1, p2); + Assertions.assertEquals(p1, p2); } } diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestMixedFormatCatalogUtil.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestMixedFormatCatalogUtil.java index 9bc1662aa4..6c414c44a8 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestMixedFormatCatalogUtil.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestMixedFormatCatalogUtil.java @@ -23,8 +23,8 @@ import org.apache.amoro.table.TableProperties; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.rest.RESTCatalog; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.util.HashMap; import java.util.Map; @@ -56,7 +56,7 @@ public void testMergeCatalogPropertiesToTable() { Map result = MixedFormatCatalogUtil.mergeCatalogPropertiesToTable(userDefined, catalogProperties); - Assert.assertEquals(expected, result); + Assertions.assertEquals(expected, result); } /** when log-store flag is off, remove all related props */ @@ -77,7 +77,7 @@ public void testMergeCatalogPropertiesToTable1() { Map result = MixedFormatCatalogUtil.mergeCatalogPropertiesToTable(userDefined, catalogProperties); - Assert.assertEquals(expected, result); + Assertions.assertEquals(expected, result); } /** user-defined prop should not be overwritten by default props */ @@ -102,7 +102,7 @@ public void testMergeCatalogPropertiesToTable2() { Map result = MixedFormatCatalogUtil.mergeCatalogPropertiesToTable(userDefined, catalogProperties); - Assert.assertEquals(expected, result); + Assertions.assertEquals(expected, result); } /** Other user-defined prop should not lose */ @@ -129,7 +129,7 @@ public void testMergeCatalogPropertiesToTable3() { Map result = MixedFormatCatalogUtil.mergeCatalogPropertiesToTable(userDefined, catalogProperties); - Assert.assertEquals(expected, result); + Assertions.assertEquals(expected, result); } /** @@ -166,7 +166,7 @@ public void testMergeCatalogPropertiesToTable4() { Map result = MixedFormatCatalogUtil.mergeCatalogPropertiesToTable(userDefined, catalogProperties); - Assert.assertEquals(expected, result); + Assertions.assertEquals(expected, result); } /** @@ -204,7 +204,7 @@ public void testMergeCatalogPropertiesToTable5() { Map result = MixedFormatCatalogUtil.mergeCatalogPropertiesToTable(userDefined, catalogProperties); - Assert.assertEquals(expected, result); + Assertions.assertEquals(expected, result); } /** @@ -237,7 +237,7 @@ public void testMergeCatalogPropertiesToTable6() { Map result = MixedFormatCatalogUtil.mergeCatalogPropertiesToTable(userDefined, catalogProperties); - Assert.assertEquals(expected, result); + Assertions.assertEquals(expected, result); } /** @@ -268,7 +268,7 @@ public void testMergeCatalogPropertiesToTable7() { Map result = MixedFormatCatalogUtil.mergeCatalogPropertiesToTable(userDefined, catalogProperties); - Assert.assertEquals(expected, result); + Assertions.assertEquals(expected, result); } @Test @@ -287,7 +287,7 @@ public void testWithIcebergCatalogInitializeProperties() { props = MixedFormatCatalogUtil.withIcebergCatalogInitializeProperties( name, typeHadoop, ImmutableMap.of(keyWarehouse, path)); - Assert.assertEquals(typeHadoop, props.get(type)); + Assertions.assertEquals(typeHadoop, props.get(type)); // custom props = @@ -295,9 +295,9 @@ public void testWithIcebergCatalogInitializeProperties() { name, typeCustom, ImmutableMap.of(keyWarehouse, path, CatalogProperties.CATALOG_IMPL, restImpl)); - Assert.assertFalse(props.containsKey(type)); + Assertions.assertFalse(props.containsKey(type)); // custom args check - Assert.assertThrows( + Assertions.assertThrows( IllegalArgumentException.class, () -> { MixedFormatCatalogUtil.withIcebergCatalogInitializeProperties( @@ -308,9 +308,9 @@ public void testWithIcebergCatalogInitializeProperties() { props = MixedFormatCatalogUtil.withIcebergCatalogInitializeProperties( name, typeAms, ImmutableMap.of(keyWarehouse, path)); - Assert.assertEquals(name, props.get(keyWarehouse)); - Assert.assertFalse(props.containsKey(type)); - Assert.assertEquals(restImpl, props.get(CatalogProperties.CATALOG_IMPL)); + Assertions.assertEquals(name, props.get(keyWarehouse)); + Assertions.assertFalse(props.containsKey(type)); + Assertions.assertEquals(restImpl, props.get(CatalogProperties.CATALOG_IMPL)); } /** test merge writable properties basic case */ @@ -341,7 +341,7 @@ public void testMergeDefaultWritableProperties1() { Map result = MixedFormatCatalogUtil.mergePersistedCatalogPropertiesToTable( tableProperties, catalogProperties); - Assert.assertEquals(expected, result); + Assertions.assertEquals(expected, result); } /** test handling log-store related properties */ @@ -362,7 +362,7 @@ public void testMergeDefaultWritableProperties2() { Map result = MixedFormatCatalogUtil.mergePersistedCatalogPropertiesToTable( tableProperties, catalogProperties); - Assert.assertEquals(expected, result); + Assertions.assertEquals(expected, result); } /** test merge additional properties */ @@ -391,7 +391,7 @@ public void testMergeDefaultWritableProperties3() { MixedFormatCatalogUtil.mergePersistedCatalogPropertiesToTable( tableProperties, catalogProperties); - Assert.assertEquals(expected, result); + Assertions.assertEquals(expected, result); } /** test merge additional props */ @@ -419,7 +419,7 @@ public void testMergeDefaultWritableProperties4() { Map result = MixedFormatCatalogUtil.mergePersistedCatalogPropertiesToTable( tableProperties, catalogProperties); - Assert.assertEquals(expected, result); + Assertions.assertEquals(expected, result); } /** test merge additional prefix */ @@ -446,7 +446,7 @@ public void testMergeDefaultWritableProperties5() { MixedFormatCatalogUtil.mergePersistedCatalogPropertiesToTable( tableProperties, catalogProperties); - Assert.assertEquals(expected, result); + Assertions.assertEquals(expected, result); } /** test merge excluded props */ @@ -470,7 +470,7 @@ public void testMergeDefaultWritableProperties6() { Map result = MixedFormatCatalogUtil.mergePersistedCatalogPropertiesToTable( tableProperties, catalogProperties); - Assert.assertEquals(expected, result); + Assertions.assertEquals(expected, result); } /** test merge excluded prefix */ @@ -496,7 +496,7 @@ public void testMergeDefaultWritableProperties7() { Map result = MixedFormatCatalogUtil.mergePersistedCatalogPropertiesToTable( tableProperties, catalogProperties); - Assert.assertEquals(expected, result); + Assertions.assertEquals(expected, result); } /** test merge excluded prefix */ @@ -520,7 +520,7 @@ public void testMergeDefaultWritableProperties8() { Map result = MixedFormatCatalogUtil.mergePersistedCatalogPropertiesToTable( tableProperties, catalogProperties); - Assert.assertEquals(expected, result); + Assertions.assertEquals(expected, result); } /** test conflicts between excluded and additional config prop which is a default not-writable */ @@ -540,7 +540,7 @@ public void testMergeDefaultWritableProperties9() { Map result = MixedFormatCatalogUtil.mergePersistedCatalogPropertiesToTable( tableProperties, catalogProperties); - Assert.assertEquals(expected, result); + Assertions.assertEquals(expected, result); } /** test conflicts between excluded and additional config prop which is a default not-writable */ @@ -563,7 +563,7 @@ public void testMergeDefaultWritableProperties10() { Map result = MixedFormatCatalogUtil.mergePersistedCatalogPropertiesToTable( tableProperties, catalogProperties); - Assert.assertEquals(expected, result); + Assertions.assertEquals(expected, result); } /** test default non-persisted mixed-hive properties */ @@ -579,6 +579,6 @@ public void testMergeDefaultWritableProperties11() { Map result = MixedFormatCatalogUtil.mergePersistedCatalogPropertiesToTable( tableProperties, catalogProperties); - Assert.assertEquals(expected, result); + Assertions.assertEquals(expected, result); } } diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestStatisticsFileUtil.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestStatisticsFileUtil.java index dbb9585982..d42ded485b 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestStatisticsFileUtil.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestStatisticsFileUtil.java @@ -33,33 +33,37 @@ import org.apache.iceberg.StructLike; import org.apache.iceberg.Table; import org.apache.iceberg.util.StructLikeMap; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import java.io.IOException; import java.util.List; +import java.util.stream.Stream; -@RunWith(Parameterized.class) public class TestStatisticsFileUtil extends TableTestBase { - @Parameterized.Parameters(name = "{0}, {1}") - public static Object[] parameters() { - return new Object[][] { - {new BasicCatalogTestHelper(TableFormat.ICEBERG), new BasicTableTestHelper(false, true)}, - {new BasicCatalogTestHelper(TableFormat.ICEBERG), new BasicTableTestHelper(false, false)}, - {new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), new BasicTableTestHelper(true, true)}, - {new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), new BasicTableTestHelper(true, false)} - }; + public static Stream parameters() { + return Stream.of( + Arguments.of( + new BasicCatalogTestHelper(TableFormat.ICEBERG), new BasicTableTestHelper(false, true)), + Arguments.of( + new BasicCatalogTestHelper(TableFormat.ICEBERG), + new BasicTableTestHelper(false, false)), + Arguments.of( + new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), + new BasicTableTestHelper(true, true)), + Arguments.of( + new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), + new BasicTableTestHelper(true, false))); } - public TestStatisticsFileUtil( - CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) { - super(catalogTestHelper, tableTestHelper); - } - - @Test - public void testWriteAndReadPuffin() { + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("parameters") + public void testWriteAndReadPuffin( + CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) throws IOException { + setupTable(catalogTestHelper, tableTestHelper); UnkeyedTable table = getMixedTable().isKeyedTable() ? getMixedTable().asKeyedTable().baseTable() @@ -115,14 +119,14 @@ private StructLikeMap readPartitionData(Table table, String type) { List> result = StatisticsFileUtil.reader(table) .read(findValidStatisticFile(table, type), type, dataSerializer); - Assert.assertEquals(1, result.size()); + Assertions.assertEquals(1, result.size()); return result.get(0); } private void assertStructLikeEquals(StructLikeMap expected, StructLikeMap actual) { - Assert.assertEquals(expected.size(), actual.size()); + Assertions.assertEquals(expected.size(), actual.size()); for (StructLike structLike : expected.keySet()) { - Assert.assertEquals(expected.get(structLike), actual.get(structLike)); + Assertions.assertEquals(expected.get(structLike), actual.get(structLike)); } } diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestTablePropertyUtil.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestTablePropertyUtil.java index 6401546c59..d39d98aab0 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestTablePropertyUtil.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestTablePropertyUtil.java @@ -24,8 +24,8 @@ import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.StructLikeMap; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.util.Map; @@ -64,13 +64,13 @@ private static StructLikeMap> buildPartitionProperties() { @Test public void testEncodePartitionProperties() { - Assert.assertEquals( + Assertions.assertEquals( JSON_VALUE, TablePropertyUtil.encodePartitionProperties(SPEC, PARTITION_PROPERTIES)); } @Test public void testDecodePartitionProperties() { - Assert.assertEquals( + Assertions.assertEquals( PARTITION_PROPERTIES, TablePropertyUtil.decodePartitionProperties(SPEC, JSON_VALUE)); } } diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestUnkeyedExpressionUtil.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestUnkeyedExpressionUtil.java index 051631d45f..d9004b3522 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestUnkeyedExpressionUtil.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestUnkeyedExpressionUtil.java @@ -36,25 +36,19 @@ import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.types.Types; -import org.junit.Assert; -import org.junit.Assume; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; import java.util.ArrayList; import java.util.Set; +import java.util.stream.Stream; -@RunWith(Parameterized.class) public class TestUnkeyedExpressionUtil extends TableTestBase { - public TestUnkeyedExpressionUtil(TableFormat tableFormat, PartitionSpec partitionSpec) { - super( - new BasicCatalogTestHelper(tableFormat), - new BasicTableTestHelper(TABLE_SCHEMA, false, partitionSpec)); - } - public static final Schema TABLE_SCHEMA = new Schema( Types.NestedField.required(1, "id", Types.IntegerType.get()), @@ -67,36 +61,56 @@ protected MixedTable getMixedTable() { return super.getMixedTable(); } - @Parameterized.Parameters(name = "{0}, {1}") - public static Object[][] parameters() { - return new Object[][] { - {TableFormat.ICEBERG, PartitionSpec.builderFor(TABLE_SCHEMA).identity("op_time").build()}, - {TableFormat.ICEBERG, PartitionSpec.builderFor(TABLE_SCHEMA).bucket("name", 2).build()}, - {TableFormat.ICEBERG, PartitionSpec.builderFor(TABLE_SCHEMA).truncate("ts", 10).build()}, - {TableFormat.ICEBERG, PartitionSpec.builderFor(TABLE_SCHEMA).year("op_time").build()}, - {TableFormat.ICEBERG, PartitionSpec.builderFor(TABLE_SCHEMA).month("op_time").build()}, - {TableFormat.ICEBERG, PartitionSpec.builderFor(TABLE_SCHEMA).day("op_time").build()}, - {TableFormat.ICEBERG, PartitionSpec.builderFor(TABLE_SCHEMA).hour("op_time").build()}, - {TableFormat.ICEBERG, PartitionSpec.unpartitioned()}, - { - TableFormat.MIXED_ICEBERG, - PartitionSpec.builderFor(TABLE_SCHEMA).identity("op_time").build() - }, - {TableFormat.MIXED_ICEBERG, PartitionSpec.builderFor(TABLE_SCHEMA).bucket("name", 2).build()}, - { - TableFormat.MIXED_ICEBERG, PartitionSpec.builderFor(TABLE_SCHEMA).truncate("ts", 10).build() - }, - {TableFormat.MIXED_ICEBERG, PartitionSpec.builderFor(TABLE_SCHEMA).year("op_time").build()}, - {TableFormat.MIXED_ICEBERG, PartitionSpec.builderFor(TABLE_SCHEMA).month("op_time").build()}, - {TableFormat.MIXED_ICEBERG, PartitionSpec.builderFor(TABLE_SCHEMA).day("op_time").build()}, - {TableFormat.MIXED_ICEBERG, PartitionSpec.builderFor(TABLE_SCHEMA).hour("op_time").build()}, - {TableFormat.MIXED_ICEBERG, PartitionSpec.unpartitioned()} - }; + public static Stream parameters() { + return Stream.of( + Arguments.of( + TableFormat.ICEBERG, + PartitionSpec.builderFor(TABLE_SCHEMA).identity("op_time").build()), + Arguments.of( + TableFormat.ICEBERG, PartitionSpec.builderFor(TABLE_SCHEMA).bucket("name", 2).build()), + Arguments.of( + TableFormat.ICEBERG, PartitionSpec.builderFor(TABLE_SCHEMA).truncate("ts", 10).build()), + Arguments.of( + TableFormat.ICEBERG, PartitionSpec.builderFor(TABLE_SCHEMA).year("op_time").build()), + Arguments.of( + TableFormat.ICEBERG, PartitionSpec.builderFor(TABLE_SCHEMA).month("op_time").build()), + Arguments.of( + TableFormat.ICEBERG, PartitionSpec.builderFor(TABLE_SCHEMA).day("op_time").build()), + Arguments.of( + TableFormat.ICEBERG, PartitionSpec.builderFor(TABLE_SCHEMA).hour("op_time").build()), + Arguments.of(TableFormat.ICEBERG, PartitionSpec.unpartitioned()), + Arguments.of( + TableFormat.MIXED_ICEBERG, + PartitionSpec.builderFor(TABLE_SCHEMA).identity("op_time").build()), + Arguments.of( + TableFormat.MIXED_ICEBERG, + PartitionSpec.builderFor(TABLE_SCHEMA).bucket("name", 2).build()), + Arguments.of( + TableFormat.MIXED_ICEBERG, + PartitionSpec.builderFor(TABLE_SCHEMA).truncate("ts", 10).build()), + Arguments.of( + TableFormat.MIXED_ICEBERG, + PartitionSpec.builderFor(TABLE_SCHEMA).year("op_time").build()), + Arguments.of( + TableFormat.MIXED_ICEBERG, + PartitionSpec.builderFor(TABLE_SCHEMA).month("op_time").build()), + Arguments.of( + TableFormat.MIXED_ICEBERG, + PartitionSpec.builderFor(TABLE_SCHEMA).day("op_time").build()), + Arguments.of( + TableFormat.MIXED_ICEBERG, + PartitionSpec.builderFor(TABLE_SCHEMA).hour("op_time").build()), + Arguments.of(TableFormat.MIXED_ICEBERG, PartitionSpec.unpartitioned())); } - @Test - public void testUnkeyedConvertPartitionStructLikeToDataFilter() throws IOException { - Assume.assumeTrue(getMixedTable().isUnkeyedTable()); + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("parameters") + public void testUnkeyedConvertPartitionStructLikeToDataFilter( + TableFormat tableFormat, PartitionSpec partitionSpec) throws IOException { + setupTable( + new BasicCatalogTestHelper(tableFormat), + new BasicTableTestHelper(TABLE_SCHEMA, false, partitionSpec)); + Assumptions.assumeTrue(getMixedTable().isUnkeyedTable()); UnkeyedTable table = getMixedTable().asUnkeyedTable(); ArrayList records = Lists.newArrayList( @@ -132,9 +146,9 @@ private void assertPlanHalfWithPartitionFilter(Expression partitionFilter) { throw new RuntimeException(e); } if (isPartitionedTable()) { - Assert.assertEquals(2, baseDataFiles.size()); + Assertions.assertEquals(2, baseDataFiles.size()); } else { - Assert.assertEquals(1, baseDataFiles.size()); + Assertions.assertEquals(1, baseDataFiles.size()); } baseDataFiles.clear(); @@ -148,9 +162,9 @@ private void assertPlanHalfWithPartitionFilter(Expression partitionFilter) { throw new RuntimeException(e); } if (isPartitionedTable()) { - Assert.assertEquals(1, baseDataFiles.size()); + Assertions.assertEquals(1, baseDataFiles.size()); } else { - Assert.assertEquals(1, baseDataFiles.size()); + Assertions.assertEquals(1, baseDataFiles.size()); } } } diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestWatermarkGenerator.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestWatermarkGenerator.java index fe24a40c52..c62dc3b74c 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestWatermarkGenerator.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestWatermarkGenerator.java @@ -31,9 +31,11 @@ import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Types; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import java.io.IOException; import java.nio.ByteBuffer; import java.text.DateFormat; import java.text.ParseException; @@ -42,8 +44,9 @@ public class TestWatermarkGenerator extends TableTestBase { - public TestWatermarkGenerator() { - super( + @BeforeEach + public void setUp() throws IOException { + setupTable( new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), new BasicTableTestHelper(false, false)); } @@ -52,9 +55,9 @@ public TestWatermarkGenerator() { public void testDefaultEventTime() { long start = System.currentTimeMillis(); WatermarkGenerator watermarkGenerator = WatermarkGenerator.forTable(getMixedTable()); - Assert.assertEquals(-1, watermarkGenerator.watermark()); + Assertions.assertEquals(-1, watermarkGenerator.watermark()); watermarkGenerator.addFile(DataFileTestHelpers.getFile(1)); - Assert.assertTrue(watermarkGenerator.watermark() >= start); + Assertions.assertTrue(watermarkGenerator.watermark() >= start); } @Test @@ -88,7 +91,7 @@ public void testTimestampEventTime() { "/watermark", 1, PartitionSpec.unpartitioned(), null, metrics, false); watermarkGenerator.addFile(file1); - Assert.assertEquals(start - 20000, watermarkGenerator.watermark()); + Assertions.assertEquals(start - 20000, watermarkGenerator.watermark()); lowerBounds.put(4, Conversions.toByteBuffer(Types.TimestampType.withoutZone(), start)); upperBounds.put(4, Conversions.toByteBuffer(Types.TimestampType.withoutZone(), start)); @@ -106,7 +109,7 @@ public void testTimestampEventTime() { DataFileTestHelpers.getFile( "/watermark", 2, PartitionSpec.unpartitioned(), null, metrics, false); watermarkGenerator.addFile(file2); - Assert.assertEquals(start - 10000, watermarkGenerator.watermark()); + Assertions.assertEquals(start - 10000, watermarkGenerator.watermark()); } @Test @@ -140,7 +143,7 @@ public void testLongEventTime() { DataFileTestHelpers.getFile( "/watermark", 1, PartitionSpec.unpartitioned(), null, metrics, false); watermarkGenerator.addFile(file1); - Assert.assertEquals((start / 1000 * 1000) - 15000, watermarkGenerator.watermark()); + Assertions.assertEquals((start / 1000 * 1000) - 15000, watermarkGenerator.watermark()); lowerBounds.put(3, Conversions.toByteBuffer(Types.LongType.get(), start / 1000)); upperBounds.put(3, Conversions.toByteBuffer(Types.LongType.get(), start / 1000)); @@ -158,7 +161,7 @@ public void testLongEventTime() { DataFileTestHelpers.getFile( "/watermark", 2, PartitionSpec.unpartitioned(), null, metrics, false); watermarkGenerator.addFile(file2); - Assert.assertEquals((start / 1000 * 1000) - 5000, watermarkGenerator.watermark()); + Assertions.assertEquals((start / 1000 * 1000) - 5000, watermarkGenerator.watermark()); } @Test @@ -192,7 +195,8 @@ public void testStringEventTime() throws ParseException { DataFileTestHelpers.getFile( "/watermark", 1, PartitionSpec.unpartitioned(), null, metrics, false); watermarkGenerator.addFile(file1); - Assert.assertEquals(df.parse("2022-11-11 00:00:59").getTime(), watermarkGenerator.watermark()); + Assertions.assertEquals( + df.parse("2022-11-11 00:00:59").getTime(), watermarkGenerator.watermark()); lowerBounds.put(2, Conversions.toByteBuffer(Types.StringType.get(), "2022-11-11 00:00:00")); upperBounds.put(2, Conversions.toByteBuffer(Types.StringType.get(), "2022-11-11 00:02:00")); @@ -210,7 +214,8 @@ public void testStringEventTime() throws ParseException { DataFileTestHelpers.getFile( "/watermark", 2, PartitionSpec.unpartitioned(), null, metrics, false); watermarkGenerator.addFile(file2); - Assert.assertEquals(df.parse("2022-11-11 00:01:59").getTime(), watermarkGenerator.watermark()); + Assertions.assertEquals( + df.parse("2022-11-11 00:01:59").getTime(), watermarkGenerator.watermark()); } @Test @@ -242,6 +247,6 @@ public void testWithWrongConfigs() { DataFileTestHelpers.getFile( "/watermark", 1, PartitionSpec.unpartitioned(), null, metrics, false); watermarkGenerator.addFile(file1); - Assert.assertEquals(-1, watermarkGenerator.watermark()); + Assertions.assertEquals(-1, watermarkGenerator.watermark()); } } diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/map/TestRocksDBBackend.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/map/TestRocksDBBackend.java index 4945ddff4a..24b97c2593 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/map/TestRocksDBBackend.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/map/TestRocksDBBackend.java @@ -19,9 +19,9 @@ package org.apache.amoro.utils.map; import org.apache.amoro.AmoroIOException; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.io.File; import java.util.ArrayList; @@ -34,7 +34,7 @@ public class TestRocksDBBackend { public static final String CF_NAME = "TEST"; - @Before + @BeforeEach public void setup() {} @Test @@ -42,9 +42,9 @@ public void testAddAndDropColumnFamily() throws Exception { RocksDBBackend rocksDBBackend = RocksDBBackend.getOrCreateInstance(); int originalCfCount = rocksDBBackend.listColumnFamilies().size(); rocksDBBackend.addColumnFamily(CF_NAME); - Assert.assertEquals(originalCfCount + 1, rocksDBBackend.listColumnFamilies().size()); + Assertions.assertEquals(originalCfCount + 1, rocksDBBackend.listColumnFamilies().size()); rocksDBBackend.dropColumnFamily(CF_NAME); - Assert.assertEquals(originalCfCount, rocksDBBackend.listColumnFamilies().size()); + Assertions.assertEquals(originalCfCount, rocksDBBackend.listColumnFamilies().size()); } @Test @@ -54,23 +54,23 @@ public void testPutGetDelete() { rocksDBBackend.put(CF_NAME, "name", "mj"); rocksDBBackend.put(CF_NAME, 2, "zjs"); rocksDBBackend.put(CF_NAME, 4556, "zyx"); - Assert.assertEquals("zyx", rocksDBBackend.get(CF_NAME, 4556)); - Assert.assertEquals("zjs", rocksDBBackend.get(CF_NAME, 2)); - Assert.assertEquals("mj", rocksDBBackend.get(CF_NAME, "name")); + Assertions.assertEquals("zyx", rocksDBBackend.get(CF_NAME, 4556)); + Assertions.assertEquals("zjs", rocksDBBackend.get(CF_NAME, 2)); + Assertions.assertEquals("mj", rocksDBBackend.get(CF_NAME, "name")); rocksDBBackend.delete(CF_NAME, 4556); rocksDBBackend.delete(CF_NAME, "name"); - Assert.assertNull(rocksDBBackend.get(CF_NAME, 4556)); - Assert.assertNull(rocksDBBackend.get(CF_NAME, "name")); + Assertions.assertNull(rocksDBBackend.get(CF_NAME, 4556)); + Assertions.assertNull(rocksDBBackend.get(CF_NAME, "name")); rocksDBBackend.put(CF_NAME, 2, "mj"); - Assert.assertEquals("mj", rocksDBBackend.get(CF_NAME, 2)); + Assertions.assertEquals("mj", rocksDBBackend.get(CF_NAME, 2)); rocksDBBackend.put(CF_NAME, "name", "mj"); - Assert.assertEquals("mj", rocksDBBackend.get(CF_NAME, "name")); + Assertions.assertEquals("mj", rocksDBBackend.get(CF_NAME, "name")); rocksDBBackend.dropColumnFamily(CF_NAME); try { rocksDBBackend.get(CF_NAME, "name"); - Assert.fail(); + Assertions.fail(); } catch (Throwable t) { - Assert.assertTrue(t instanceof AmoroIOException); + Assertions.assertTrue(t instanceof AmoroIOException); } } @@ -89,8 +89,8 @@ public void testIterator() { } Collections.sort(expect); Collections.sort(valueList); - Assert.assertEquals(expect.size(), valueList.size()); - Assert.assertArrayEquals(expect.toArray(), valueList.toArray()); + Assertions.assertEquals(expect.size(), valueList.size()); + Assertions.assertArrayEquals(expect.toArray(), valueList.toArray()); rocksDBBackend.delete(CF_NAME, "name"); valueList = new ArrayList<>(); @@ -98,7 +98,7 @@ public void testIterator() { while (values.hasNext()) { valueList.add(values.next()); } - Assert.assertEquals(2, valueList.size()); + Assertions.assertEquals(2, valueList.size()); rocksDBBackend.dropColumnFamily(CF_NAME); } @@ -106,9 +106,9 @@ public void testIterator() { public void testClose() { RocksDBBackend rocksDBBackend = RocksDBBackend.getOrCreateInstance(); File baseFile = new File(rocksDBBackend.getRocksDBBasePath()); - Assert.assertTrue(baseFile.exists()); - Assert.assertTrue(baseFile.isDirectory()); + Assertions.assertTrue(baseFile.exists()); + Assertions.assertTrue(baseFile.isDirectory()); rocksDBBackend.close(); - Assert.assertFalse(baseFile.exists()); + Assertions.assertFalse(baseFile.exists()); } } diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/map/TestSimpleSpillableMap.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/map/TestSimpleSpillableMap.java index 3edc570404..71409f073a 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/map/TestSimpleSpillableMap.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/map/TestSimpleSpillableMap.java @@ -21,9 +21,9 @@ import org.apache.amoro.shade.guava32.com.google.common.collect.Maps; import org.apache.amoro.shade.guava32.com.google.common.collect.Sets; import org.apache.lucene.util.RamUsageEstimator; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.io.Serializable; import java.util.Map; @@ -38,7 +38,7 @@ public class TestSimpleSpillableMap { private long keySize; private long valueSize; - @Before + @BeforeEach public void initSizes() { keySize = RamUsageEstimator.sizeOfObject(new Key(), 0); valueSize = RamUsageEstimator.sizeOfObject(new Value(), 0); @@ -47,21 +47,21 @@ public void initSizes() { @Test public void testMemoryMap() { SimpleSpillableMap map = testMap(10, 10); - Assert.assertEquals(0, map.getSizeOfFileOnDiskInBytes()); + Assertions.assertEquals(0, map.getSizeOfFileOnDiskInBytes()); map.close(); } @Test public void testSpilledMap() { SimpleSpillableMap map = testMap(0, 20); - Assert.assertTrue(map.getSizeOfFileOnDiskInBytes() > 0); + Assertions.assertTrue(map.getSizeOfFileOnDiskInBytes() > 0); map.close(); } @Test public void testSpillableMap() { SimpleSpillableMap map = testMap(10, 20); - Assert.assertTrue(map.getSizeOfFileOnDiskInBytes() > 0); + Assertions.assertTrue(map.getSizeOfFileOnDiskInBytes() > 0); map.close(); } @@ -81,8 +81,8 @@ public void testSpillableMapConsistency() { expectedMap.put(key, value); actualMap.put(key, value); } - Assert.assertTrue(actualMap.getSizeOfFileOnDiskInBytes() > 0); - Assert.assertTrue(actualMap.getMemoryMapSize() < expectedMap.size()); + Assertions.assertTrue(actualMap.getSizeOfFileOnDiskInBytes() > 0); + Assertions.assertTrue(actualMap.getMemoryMapSize() < expectedMap.size()); assertSimpleMaps(actualMap, expectedMap); // update new value @@ -123,15 +123,15 @@ public void testSpillableMapRePut() { // remove k1 from memory actualMap.delete(k1); - Assert.assertEquals(v2, actualMap.get(k2)); + Assertions.assertEquals(v2, actualMap.get(k2)); // put a new value for k2 Value v3 = new Value(); actualMap.put(k2, v3); - Assert.assertEquals(v3, actualMap.get(k2)); + Assertions.assertEquals(v3, actualMap.get(k2)); actualMap.delete(k2); // should not exist in memory or on disk - Assert.assertNull(actualMap.get(k2)); + Assertions.assertNull(actualMap.get(k2)); } private SimpleSpillableMap testMap(long expectMemorySize, int expectKeyCount) { @@ -141,7 +141,7 @@ private SimpleSpillableMap testMap(long expectMemorySize, int expect null, new DefaultSizeEstimator<>(), new DefaultSizeEstimator<>()); - Assert.assertEquals(0, actualMap.getSizeOfFileOnDiskInBytes()); + Assertions.assertEquals(0, actualMap.getSizeOfFileOnDiskInBytes()); Map expectedMap = Maps.newHashMap(); for (int i = 0; i < expectKeyCount; i++) { Key key = new Key(); @@ -150,15 +150,15 @@ private SimpleSpillableMap testMap(long expectMemorySize, int expect actualMap.put(key, value); } assertSimpleMaps(actualMap, expectedMap); - Assert.assertEquals(expectMemorySize, actualMap.getMemoryMapSize()); - Assert.assertEquals( + Assertions.assertEquals(expectMemorySize, actualMap.getMemoryMapSize()); + Assertions.assertEquals( expectMemorySize * (keySize + valueSize), actualMap.getMemoryMapSpaceSize()); return actualMap; } private void assertSimpleMaps(SimpleMap actualMap, Map expectedMap) { for (K key : expectedMap.keySet()) { - Assert.assertEquals(expectedMap.get(key), actualMap.get(key)); + Assertions.assertEquals(expectedMap.get(key), actualMap.get(key)); } } diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/map/TestSimpleSpilledMap.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/map/TestSimpleSpilledMap.java index cc77f6a92f..120190a9b4 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/map/TestSimpleSpilledMap.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/map/TestSimpleSpilledMap.java @@ -19,10 +19,10 @@ package org.apache.amoro.utils.map; import org.apache.amoro.utils.SerializationUtil; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.io.Serializable; @@ -30,7 +30,7 @@ public class TestSimpleSpilledMap { private SimpleSpillableMap.SimpleSpilledMap map; - @Before + @BeforeEach public void createMap() { SimpleSpillableMap spillableMap = new SimpleSpillableMap( @@ -43,7 +43,7 @@ public void createMap() { null); } - @After + @AfterEach public void disposeMap() { map.close(); map = null; @@ -57,14 +57,14 @@ public void testPutGetRemove() { map.put(2, "zjs"); map.put(4556, "zyx"); map.put(key, value); - Assert.assertEquals(555, map.get("name")); - Assert.assertEquals("zjs", map.get(2)); - Assert.assertEquals("zyx", map.get(4556)); - Assert.assertEquals(value, map.get(key)); + Assertions.assertEquals(555, map.get("name")); + Assertions.assertEquals("zjs", map.get(2)); + Assertions.assertEquals("zyx", map.get(4556)); + Assertions.assertEquals(value, map.get(key)); map.delete(4556); - Assert.assertNull(map.get(4556)); + Assertions.assertNull(map.get(4556)); map.put(4556, value); - Assert.assertEquals(value, map.get(4556)); + Assertions.assertEquals(value, map.get(4556)); } @Test @@ -72,9 +72,9 @@ public void testPutNull() { Key key = new Key(); Value value = new Value(); map.put(key, value); - Assert.assertEquals(value, map.get(key)); - Assert.assertThrows(Exception.class, () -> map.put(key, null)); - Assert.assertThrows(Exception.class, () -> map.put(null, value)); + Assertions.assertEquals(value, map.get(key)); + Assertions.assertThrows(Exception.class, () -> map.put(key, null)); + Assertions.assertThrows(Exception.class, () -> map.put(null, value)); } private static class Key implements Serializable { diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/map/TestStructLikeMap.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/map/TestStructLikeMap.java index 2a3cb453d1..a281acd61e 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/map/TestStructLikeMap.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/map/TestStructLikeMap.java @@ -24,8 +24,8 @@ import org.apache.iceberg.types.Types; import org.apache.iceberg.util.StructLikeMap; import org.apache.iceberg.util.StructProjection; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -81,7 +81,7 @@ private void testMap(StructLikeBaseMap actualMap) throws IOException for (long i = 0; i < count; i++) { StructLike data = new DataStructLike(); StructLike key = StructProjection.create(DATA_SCHEMA, PK_SCHEMA).copyFor(data); - Assert.assertEquals(expectedMap.get(key), actualMap.get(key)); + Assertions.assertEquals(expectedMap.get(key), actualMap.get(key)); } actualMap.close(); } diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/map/TestStructLikeWrapperSizeEstimator.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/map/TestStructLikeWrapperSizeEstimator.java index eb45d8e5a5..35a3ed1b7a 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/map/TestStructLikeWrapperSizeEstimator.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/utils/map/TestStructLikeWrapperSizeEstimator.java @@ -28,8 +28,8 @@ import org.apache.iceberg.types.Types; import org.apache.iceberg.util.StructLikeWrapper; import org.apache.lucene.util.RamUsageEstimator; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.util.Map; @@ -53,7 +53,7 @@ public void testSizeEstimator() { // Because the size of map also will increase, so the record2Size should a little bigger than // the size of the record long estimateSize = new StructLikeWrapperSizeEstimator().sizeEstimate(wrapper); - Assert.assertEquals(1, record2Size / estimateSize); + Assertions.assertEquals(1, record2Size / estimateSize); } @Test @@ -78,6 +78,6 @@ public void testSizeEstimatorWithNullField() { StructLikeWrapper wrapper = StructLikeWrapper.forType(schema.asStruct()).set(record2); long estimateSize = new StructLikeWrapperSizeEstimator().sizeEstimate(wrapper); - Assert.assertEquals(1, record2Size / estimateSize); + Assertions.assertEquals(1, record2Size / estimateSize); } } diff --git a/amoro-format-iceberg/src/test/java/org/apache/iceberg/TestIcebergFindFiles.java b/amoro-format-iceberg/src/test/java/org/apache/iceberg/TestIcebergFindFiles.java index e7fa4a37e2..cecc514c02 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/iceberg/TestIcebergFindFiles.java +++ b/amoro-format-iceberg/src/test/java/org/apache/iceberg/TestIcebergFindFiles.java @@ -25,56 +25,35 @@ import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Types; -import org.junit.After; -import org.junit.Assert; -import org.junit.Assume; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.api.extension.ExtendWith; import java.util.Arrays; import java.util.List; import java.util.Set; -@RunWith(Parameterized.class) +@ExtendWith(ParameterizedTestExtension.class) public class TestIcebergFindFiles extends TestBase { - @Parameterized.Parameters(name = "formatVersion = {0}") + @Parameters(name = "formatVersion = {0}") public static List parameters() { return Arrays.asList(1, 2); } - @Rule public TemporaryFolder tempDir = new TemporaryFolder(); + // TestBase's @BeforeEach setupTable() and @AfterEach cleanupTables() already fire. - public TestIcebergFindFiles(int formatVersion) { - this.formatVersion = formatVersion; - } - - @Before - public void setUp() throws Exception { - this.tableDir = tempDir.newFolder(); - this.tableDir.delete(); - super.setupTable(); - } - - @After - public void cleanUp() throws Exception { - super.cleanupTables(); - } - - @Test + @TestTemplate public void testBasicBehavior() { table.newAppend().appendFile(FILE_A).appendFile(FILE_B).commit(); Iterable> files = transform(new IcebergFindFiles(table).entries()); - Assert.assertEquals(pathSet(FILE_A, FILE_B), pathSet(files)); + Assertions.assertEquals(pathSet(FILE_A, FILE_B), pathSet(files)); } - @Test + @TestTemplate public void testWithMetadataMatching() { table .newAppend() @@ -89,10 +68,10 @@ public void testWithMetadataMatching() { new IcebergFindFiles(table) .filterFiles(Expressions.startsWith("file_path", "/path/to/data-a")) .entries()); - Assert.assertEquals(pathSet(FILE_A), pathSet(files)); + Assertions.assertEquals(pathSet(FILE_A), pathSet(files)); } - @Test + @TestTemplate public void testWithRecordsMatching() { table .newAppend() @@ -119,10 +98,10 @@ public void testWithRecordsMatching() { final Iterable> files = transform(new IcebergFindFiles(table).filterData(Expressions.equal("id", 1)).entries()); - Assert.assertEquals(Sets.newHashSet("/path/to/data-e.parquet"), pathSet(files)); + Assertions.assertEquals(Sets.newHashSet("/path/to/data-e.parquet"), pathSet(files)); } - @Test + @TestTemplate public void testInPartition() { table .newAppend() @@ -140,10 +119,10 @@ public void testInPartition() { Lists.newArrayList(StaticDataTask.Row.of(1), StaticDataTask.Row.of(2))) .entries()); - Assert.assertEquals(pathSet(FILE_B, FILE_C), pathSet(files)); + Assertions.assertEquals(pathSet(FILE_B, FILE_C), pathSet(files)); } - @Test + @TestTemplate public void testInPartitionForRemovePartitionField() { table .newAppend() @@ -173,7 +152,7 @@ public void testInPartitionForRemovePartitionField() { Lists.newArrayList(StaticDataTask.Row.of(1), StaticDataTask.Row.of(2))) .entries()); - Assert.assertEquals(pathSet(FILE_B, FILE_C), pathSet(files)); + Assertions.assertEquals(pathSet(FILE_B, FILE_C), pathSet(files)); Iterable> files2 = transform( @@ -181,10 +160,10 @@ public void testInPartitionForRemovePartitionField() { .inPartitions(table.spec(), StaticDataTask.Row.of(new Object[] {null})) .entries()); - Assert.assertEquals(pathSet(fileDropPartitionField), pathSet(files2)); + Assertions.assertEquals(pathSet(fileDropPartitionField), pathSet(files2)); } - @Test + @TestTemplate public void testInPartitionForAddPartitionField() { table .newAppend() @@ -214,7 +193,7 @@ public void testInPartitionForAddPartitionField() { table.specs().get(0), StaticDataTask.Row.of(1), StaticDataTask.Row.of(2)) .entries()); - Assert.assertEquals(pathSet(FILE_B, FILE_C), pathSet(files)); + Assertions.assertEquals(pathSet(FILE_B, FILE_C), pathSet(files)); Iterable> files2 = transform( @@ -222,12 +201,12 @@ public void testInPartitionForAddPartitionField() { .inPartitions(table.specs().get(1), StaticDataTask.Row.of(0, 1)) .entries()); - Assert.assertEquals(pathSet(fileAddPartitionField), pathSet(files2)); + Assertions.assertEquals(pathSet(fileAddPartitionField), pathSet(files2)); } - @Test + @TestTemplate public void testInPartitionForReplacePartitionField() { - Assume.assumeTrue(formatVersion == 2); + Assumptions.assumeTrue(formatVersion == 2); table .newAppend() .appendFile(FILE_A) // bucket 0 @@ -261,7 +240,7 @@ public void testInPartitionForReplacePartitionField() { table.specs().get(0), StaticDataTask.Row.of(1), StaticDataTask.Row.of(2)) .entries()); - Assert.assertEquals(pathSet(FILE_B, FILE_C), pathSet(files)); + Assertions.assertEquals(pathSet(FILE_B, FILE_C), pathSet(files)); Iterable> files2 = transform( @@ -269,10 +248,10 @@ public void testInPartitionForReplacePartitionField() { .inPartitions(table.specs().get(1), StaticDataTask.Row.of(1)) .entries()); - Assert.assertEquals(pathSet(fileReplacePartitionField), pathSet(files2)); + Assertions.assertEquals(pathSet(fileReplacePartitionField), pathSet(files2)); } - @Test + @TestTemplate public void testAsOfTimestamp() { table.newAppend().appendFile(FILE_A).commit(); @@ -287,10 +266,10 @@ public void testAsOfTimestamp() { Iterable> files = transform(new IcebergFindFiles(table).asOfTime(timestamp).entries()); - Assert.assertEquals(pathSet(FILE_A, FILE_B), pathSet(files)); + Assertions.assertEquals(pathSet(FILE_A, FILE_B), pathSet(files)); } - @Test + @TestTemplate public void testSnapshotId() { table.newAppend().appendFile(FILE_A).appendFile(FILE_B).commit(); @@ -303,10 +282,10 @@ public void testSnapshotId() { Iterable> files = transform(new IcebergFindFiles(table).inSnapshot(snapshotId).entries()); - Assert.assertEquals(pathSet(FILE_A, FILE_B, FILE_C), pathSet(files)); + Assertions.assertEquals(pathSet(FILE_A, FILE_B, FILE_C), pathSet(files)); } - @Test + @TestTemplate public void testCaseSensitivity() { table .newAppend() @@ -323,10 +302,10 @@ public void testCaseSensitivity() { .filterFiles(Expressions.startsWith("FILE_PATH", "/path/to/data-a")) .entries()); - Assert.assertEquals(pathSet(FILE_A), pathSet(files)); + Assertions.assertEquals(pathSet(FILE_A), pathSet(files)); } - @Test + @TestTemplate public void testIncludeColumnStats() { table.newAppend().appendFile(FILE_WITH_STATS).commit(); @@ -334,32 +313,32 @@ public void testIncludeColumnStats() { transform(new IcebergFindFiles(table).includeColumnStats().entries()); final ContentFile file = files.iterator().next(); - Assert.assertEquals(FILE_WITH_STATS.columnSizes(), file.columnSizes()); - Assert.assertEquals(FILE_WITH_STATS.valueCounts(), file.valueCounts()); - Assert.assertEquals(FILE_WITH_STATS.nullValueCounts(), file.nullValueCounts()); - Assert.assertEquals(FILE_WITH_STATS.nanValueCounts(), file.nanValueCounts()); - Assert.assertEquals(FILE_WITH_STATS.lowerBounds(), file.lowerBounds()); - Assert.assertEquals(FILE_WITH_STATS.upperBounds(), file.upperBounds()); + Assertions.assertEquals(FILE_WITH_STATS.columnSizes(), file.columnSizes()); + Assertions.assertEquals(FILE_WITH_STATS.valueCounts(), file.valueCounts()); + Assertions.assertEquals(FILE_WITH_STATS.nullValueCounts(), file.nullValueCounts()); + Assertions.assertEquals(FILE_WITH_STATS.nanValueCounts(), file.nanValueCounts()); + Assertions.assertEquals(FILE_WITH_STATS.lowerBounds(), file.lowerBounds()); + Assertions.assertEquals(FILE_WITH_STATS.upperBounds(), file.upperBounds()); } - @Test + @TestTemplate public void testFileContent() { - Assume.assumeTrue(formatVersion == 2); + Assumptions.assumeTrue(formatVersion == 2); table.newRowDelta().addRows(FILE_A).addDeletes(FILE_A_DELETES).commit(); Iterable> allFiles = transform(new IcebergFindFiles(table).entries()); - Assert.assertEquals(pathSet(FILE_A, FILE_A_DELETES), pathSet(allFiles)); + Assertions.assertEquals(pathSet(FILE_A, FILE_A_DELETES), pathSet(allFiles)); Iterable> dataFiles = transform(new IcebergFindFiles(table).fileContent(ManifestContent.DATA).entries()); - Assert.assertEquals(pathSet(FILE_A), pathSet(dataFiles)); + Assertions.assertEquals(pathSet(FILE_A), pathSet(dataFiles)); Iterable> deleteFiles = transform(new IcebergFindFiles(table).fileContent(ManifestContent.DELETES).entries()); - Assert.assertEquals(pathSet(FILE_A_DELETES), pathSet(deleteFiles)); + Assertions.assertEquals(pathSet(FILE_A_DELETES), pathSet(deleteFiles)); } - @Test + @TestTemplate public void testNoSnapshot() { // a table has no snapshot when it just gets created and no data is loaded yet @@ -367,7 +346,7 @@ public void testNoSnapshot() { Iterable files = new IcebergFindFiles(table).entries(); // verify an empty collection of data file is returned - Assert.assertEquals(0, Sets.newHashSet(files).size()); + Assertions.assertEquals(0, Sets.newHashSet(files).size()); } private Set pathSet(ContentFile... files) {