Skip to content
Permalink
Browse files
[FLINK-27323][table][tests] Migrate table-api-java to JUnit5
  • Loading branch information
snuyanzin committed May 16, 2022
1 parent 0ff1e40 commit a4fa2857c371c9ee9656357390d0367896ad6c90
Showing 27 changed files with 568 additions and 518 deletions.
@@ -37,29 +37,28 @@
import org.apache.hadoop.hive.metastore.api.FunctionType;
import org.apache.hadoop.hive.metastore.api.PrincipalType;
import org.apache.hadoop.hive.metastore.api.Table;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Map;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.assertj.core.api.Assertions.assertThat;

/** Test for HiveCatalog on generic metadata. */
public class HiveCatalogGenericMetadataTest extends HiveCatalogMetadataTestBase {
class HiveCatalogGenericMetadataTest extends HiveCatalogMetadataTestBase {

@BeforeClass
public static void init() {
@BeforeAll
static void init() {
catalog = HiveTestUtils.createHiveCatalog();
catalog.open();
}

// ------ tables ------

@Test
public void testGenericTableSchema() throws Exception {
void testGenericTableSchema() throws Exception {
catalog.createDatabase(db1, createDb(), false);

TableSchema tableSchema =
@@ -81,15 +80,15 @@ public void testGenericTableSchema() throws Exception {
new CatalogTableImpl(tableSchema, getBatchTableProperties(), TEST_COMMENT),
false);

assertEquals(tableSchema, catalog.getTable(tablePath).getSchema());
assertThat(catalog.getTable(tablePath).getSchema()).isEqualTo(tableSchema);
} finally {
catalog.dropTable(tablePath, true);
}
}

@Test
// NOTE: Be careful to modify this test, it is important to backward compatibility
public void testTableSchemaCompatibility() throws Exception {
void testTableSchemaCompatibility() throws Exception {
catalog.createDatabase(db1, createDb(), false);
try {
// table with numeric types
@@ -121,7 +120,7 @@ public void testTableSchemaCompatibility() throws Exception {
hiveTable.getParameters().put("flink.generic.table.schema.7.data-type", "DOUBLE");
((HiveCatalog) catalog).client.createTable(hiveTable);
CatalogBaseTable catalogBaseTable = catalog.getTable(tablePath);
assertFalse(HiveCatalog.isHiveTable(catalogBaseTable.getOptions()));
assertThat(HiveCatalog.isHiveTable(catalogBaseTable.getOptions())).isFalse();
TableSchema expectedSchema =
TableSchema.builder()
.fields(
@@ -137,7 +136,7 @@ public void testTableSchemaCompatibility() throws Exception {
})
.field("cost", DataTypes.DOUBLE(), "`d` * `bi`")
.build();
assertEquals(expectedSchema, catalogBaseTable.getSchema());
assertThat(catalogBaseTable.getSchema()).isEqualTo(expectedSchema);

// table with character types
tablePath = new ObjectPath(db1, "generic2");
@@ -187,7 +186,7 @@ public void testTableSchemaCompatibility() throws Exception {
})
.field("len", DataTypes.INT(), "CHAR_LENGTH(`s`)")
.build();
assertEquals(expectedSchema, catalogBaseTable.getSchema());
assertThat(catalogBaseTable.getSchema()).isEqualTo(expectedSchema);

// table with date/time types
tablePath = new ObjectPath(db1, "generic3");
@@ -233,7 +232,7 @@ public void testTableSchemaCompatibility() throws Exception {
})
.watermark("ts", "ts", DataTypes.TIMESTAMP(3))
.build();
assertEquals(expectedSchema, catalogBaseTable.getSchema());
assertThat(catalogBaseTable.getSchema()).isEqualTo(expectedSchema);

// table with complex/misc types
tablePath = new ObjectPath(db1, "generic4");
@@ -293,14 +292,14 @@ public void testTableSchemaCompatibility() throws Exception {
})
.watermark("ts", "`ts` - INTERVAL '5' SECOND", DataTypes.TIMESTAMP(3))
.build();
assertEquals(expectedSchema, catalogBaseTable.getSchema());
assertThat(catalogBaseTable.getSchema()).isEqualTo(expectedSchema);
} finally {
catalog.dropDatabase(db1, true, true);
}
}

@Test
public void testFunctionCompatibility() throws Exception {
void testFunctionCompatibility() throws Exception {
catalog.createDatabase(db1, createDb(), false);
// create a function with old prefix 'flink:' and make sure we can properly retrieve it
((HiveCatalog) catalog)
@@ -315,12 +314,12 @@ public void testFunctionCompatibility() throws Exception {
FunctionType.JAVA,
new ArrayList<>()));
CatalogFunction catalogFunction = catalog.getFunction(path1);
assertEquals("class.name", catalogFunction.getClassName());
assertEquals(FunctionLanguage.JAVA, catalogFunction.getFunctionLanguage());
assertThat(catalogFunction.getClassName()).isEqualTo("class.name");
assertThat(catalogFunction.getFunctionLanguage()).isEqualTo(FunctionLanguage.JAVA);
}

@Test
public void testGenericTableWithoutConnectorProp() throws Exception {
void testGenericTableWithoutConnectorProp() throws Exception {
catalog.createDatabase(db1, createDb(), false);
TableSchema tableSchema =
TableSchema.builder()
@@ -332,38 +331,38 @@ public void testGenericTableWithoutConnectorProp() throws Exception {
CatalogTable catalogTable = new CatalogTableImpl(tableSchema, Collections.emptyMap(), null);
catalog.createTable(path1, catalogTable, false);
CatalogTable retrievedTable = (CatalogTable) catalog.getTable(path1);
assertEquals(tableSchema, retrievedTable.getSchema());
assertEquals(Collections.emptyMap(), retrievedTable.getOptions());
assertThat(retrievedTable.getSchema()).isEqualTo(tableSchema);
assertThat(retrievedTable.getOptions()).isEmpty();
}

// ------ functions ------

@Test
public void testFunctionWithNonExistClass() throws Exception {
void testFunctionWithNonExistClass() throws Exception {
// to make sure hive catalog doesn't check function class
catalog.createDatabase(db1, createDb(), false);
CatalogFunction catalogFunction =
new CatalogFunctionImpl("non.exist.scala.class", FunctionLanguage.SCALA);
catalog.createFunction(path1, catalogFunction, false);
assertEquals(catalogFunction.getClassName(), catalog.getFunction(path1).getClassName());
assertEquals(
catalogFunction.getFunctionLanguage(),
catalog.getFunction(path1).getFunctionLanguage());
assertThat(catalog.getFunction(path1).getClassName())
.isEqualTo(catalogFunction.getClassName());
assertThat(catalog.getFunction(path1).getFunctionLanguage())
.isEqualTo(catalogFunction.getFunctionLanguage());
// alter the function
catalogFunction = new CatalogFunctionImpl("non.exist.java.class", FunctionLanguage.JAVA);
catalog.alterFunction(path1, catalogFunction, false);
assertEquals(catalogFunction.getClassName(), catalog.getFunction(path1).getClassName());
assertEquals(
catalogFunction.getFunctionLanguage(),
catalog.getFunction(path1).getFunctionLanguage());
assertThat(catalog.getFunction(path1).getClassName())
.isEqualTo(catalogFunction.getClassName());
assertThat(catalog.getFunction(path1).getFunctionLanguage())
.isEqualTo(catalogFunction.getFunctionLanguage());

catalogFunction =
new CatalogFunctionImpl("non.exist.python.class", FunctionLanguage.PYTHON);
catalog.alterFunction(path1, catalogFunction, false);
assertEquals(catalogFunction.getClassName(), catalog.getFunction(path1).getClassName());
assertEquals(
catalogFunction.getFunctionLanguage(),
catalog.getFunction(path1).getFunctionLanguage());
assertThat(catalog.getFunction(path1).getClassName())
.isEqualTo(catalogFunction.getClassName());
assertThat(catalog.getFunction(path1).getFunctionLanguage())
.isEqualTo(catalogFunction.getFunctionLanguage());
}

// ------ partitions ------
@@ -50,17 +50,15 @@
import org.apache.flink.table.factories.FactoryUtil;
import org.apache.flink.table.types.AbstractDataType;
import org.apache.flink.table.types.DataType;
import org.apache.flink.util.StringUtils;

import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.ql.udf.UDFRand;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFAbs;
import org.junit.Assume;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;

import java.util.ArrayList;
import java.util.Collections;
@@ -70,16 +68,14 @@

import static org.apache.flink.sql.parser.hive.ddl.SqlCreateHiveTable.IDENTIFIER;
import static org.apache.flink.table.factories.FactoryUtil.CONNECTOR;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assumptions.assumeThat;

/** Test for HiveCatalog on Hive metadata. */
public class HiveCatalogHiveMetadataTest extends HiveCatalogMetadataTestBase {
class HiveCatalogHiveMetadataTest extends HiveCatalogMetadataTestBase {

@BeforeClass
public static void init() {
@BeforeAll
static void init() {
catalog = HiveTestUtils.createHiveCatalog();
catalog.open();
}
@@ -92,23 +88,23 @@ public void testCreateTable_Streaming() throws Exception {}

@Test
// verifies that input/output formats and SerDe are set for Hive tables
public void testCreateTable_StorageFormatSet() throws Exception {
void testCreateTable_StorageFormatSet() throws Exception {
catalog.createDatabase(db1, createDb(), false);
catalog.createTable(path1, createTable(), false);

Table hiveTable = ((HiveCatalog) catalog).getHiveTable(path1);
String inputFormat = hiveTable.getSd().getInputFormat();
String outputFormat = hiveTable.getSd().getOutputFormat();
String serde = hiveTable.getSd().getSerdeInfo().getSerializationLib();
assertFalse(StringUtils.isNullOrWhitespaceOnly(inputFormat));
assertFalse(StringUtils.isNullOrWhitespaceOnly(outputFormat));
assertFalse(StringUtils.isNullOrWhitespaceOnly(serde));
assertThat(inputFormat).isNotBlank();
assertThat(outputFormat).isNotBlank();
assertThat(serde).isNotBlank();
}

// ------ table and column stats ------

@Test
public void testViewCompatibility() throws Exception {
void testViewCompatibility() throws Exception {
// we always store view schema via properties now
// make sure non-generic views created previously can still be used
catalog.createDatabase(db1, createDb(), false);
@@ -144,12 +140,12 @@ public void testViewCompatibility() throws Exception {

((HiveCatalog) catalog).client.createTable(hiveView);
CatalogBaseTable baseTable = catalog.getTable(path1);
assertTrue(baseTable instanceof CatalogView);
assertThat(baseTable).isInstanceOf(CatalogView.class);
CatalogView catalogView = (CatalogView) baseTable;
assertEquals(schema, catalogView.getUnresolvedSchema());
assertEquals(originQuery, catalogView.getOriginalQuery());
assertEquals(expandedQuery, catalogView.getExpandedQuery());
assertEquals("v1", catalogView.getOptions().get("k1"));
assertThat(catalogView.getUnresolvedSchema()).isEqualTo(schema);
assertThat(catalogView.getOriginalQuery()).isEqualTo(originQuery);
assertThat(catalogView.getExpandedQuery()).isEqualTo(expandedQuery);
assertThat(catalogView.getOptions().get("k1")).isEqualTo("v1");

// test mark as non-generic with connector
hiveView.setDbName(path3.getDatabaseName());
@@ -159,16 +155,16 @@ public void testViewCompatibility() throws Exception {

((HiveCatalog) catalog).client.createTable(hiveView);
baseTable = catalog.getTable(path3);
assertTrue(baseTable instanceof CatalogView);
assertThat(baseTable).isInstanceOf(CatalogView.class);
catalogView = (CatalogView) baseTable;
assertEquals(schema, catalogView.getUnresolvedSchema());
assertEquals(originQuery, catalogView.getOriginalQuery());
assertEquals(expandedQuery, catalogView.getExpandedQuery());
assertEquals("v1", catalogView.getOptions().get("k1"));
assertThat(catalogView.getUnresolvedSchema()).isEqualTo(schema);
assertThat(catalogView.getOriginalQuery()).isEqualTo(originQuery);
assertThat(catalogView.getExpandedQuery()).isEqualTo(expandedQuery);
assertThat(catalogView.getOptions().get("k1")).isEqualTo("v1");
}

@Test
public void testAlterTableColumnStatistics() throws Exception {
void testAlterTableColumnStatistics() throws Exception {
String hiveVersion = ((HiveCatalog) catalog).getHiveVersion();
boolean supportDateStats = hiveVersion.compareTo(HiveShimLoader.HIVE_VERSION_V2_3_0) >= 0;
catalog.createDatabase(db1, createDb(), false);
@@ -219,7 +215,7 @@ public void testAlterTableColumnStatistics() throws Exception {
}

@Test
public void testAlterPartitionColumnStatistics() throws Exception {
void testAlterPartitionColumnStatistics() throws Exception {
catalog.createDatabase(db1, createDb(), false);
CatalogTable catalogTable = createPartitionedTable();
catalog.createTable(path1, catalogTable, false);
@@ -246,16 +242,16 @@ public void testAlterPartitionColumnStatistics() throws Exception {
}

@Test
public void testHiveStatistics() throws Exception {
void testHiveStatistics() throws Exception {
catalog.createDatabase(db1, createDb(), false);
checkStatistics(0, -1);
checkStatistics(1, 1);
checkStatistics(1000, 1000);
}

@Test
public void testCreateTableWithConstraints() throws Exception {
Assume.assumeTrue(HiveVersionTestUtil.HIVE_310_OR_LATER);
void testCreateTableWithConstraints() throws Exception {
assumeThat(HiveVersionTestUtil.HIVE_310_OR_LATER).isTrue();
HiveCatalog hiveCatalog = (HiveCatalog) catalog;
hiveCatalog.createDatabase(db1, createDb(), false);
TableSchema.Builder builder = TableSchema.builder();
@@ -270,13 +266,16 @@ public void testCreateTableWithConstraints() throws Exception {
new CatalogTableImpl(builder.build(), getBatchTableProperties(), null),
false);
CatalogTable catalogTable = (CatalogTable) hiveCatalog.getTable(path1);
assertTrue("PK not present", catalogTable.getSchema().getPrimaryKey().isPresent());
assertThat(catalogTable.getSchema().getPrimaryKey()).as("PK not present").isPresent();
UniqueConstraint pk = catalogTable.getSchema().getPrimaryKey().get();
assertEquals("pk_name", pk.getName());
assertEquals(Collections.singletonList("x"), pk.getColumns());
assertFalse(catalogTable.getSchema().getFieldDataTypes()[0].getLogicalType().isNullable());
assertFalse(catalogTable.getSchema().getFieldDataTypes()[1].getLogicalType().isNullable());
assertTrue(catalogTable.getSchema().getFieldDataTypes()[2].getLogicalType().isNullable());
assertThat(pk.getName()).isEqualTo("pk_name");
assertThat(pk.getColumns()).isEqualTo(Collections.singletonList("x"));
assertThat(catalogTable.getSchema().getFieldDataTypes()[0].getLogicalType().isNullable())
.isFalse();
assertThat(catalogTable.getSchema().getFieldDataTypes()[1].getLogicalType().isNullable())
.isFalse();
assertThat(catalogTable.getSchema().getFieldDataTypes()[2].getLogicalType().isNullable())
.isTrue();

hiveCatalog.dropDatabase(db1, false, true);
}
@@ -288,11 +287,10 @@ public void testAlterPartition() throws Exception {
catalog.createTable(path1, createPartitionedTable(), false);
catalog.createPartition(path1, createPartitionSpec(), createPartition(), false);

assertEquals(
Collections.singletonList(createPartitionSpec()), catalog.listPartitions(path1));
assertThat(catalog.listPartitions(path1)).containsExactly(createPartitionSpec());
CatalogPartition cp = catalog.getPartition(path1, createPartitionSpec());
CatalogTestUtil.checkEquals(createPartition(), cp);
assertNull(cp.getProperties().get("k"));
assertThat(cp.getProperties().get("k")).isNull();

CatalogPartition another = createPartition();
another.getProperties().put("k", "v");
@@ -303,13 +301,12 @@ public void testAlterPartition() throws Exception {

catalog.alterPartition(path1, createPartitionSpec(), another, false);

assertEquals(
Collections.singletonList(createPartitionSpec()), catalog.listPartitions(path1));
assertThat(catalog.listPartitions(path1)).containsExactly(createPartitionSpec());

cp = catalog.getPartition(path1, createPartitionSpec());

CatalogTestUtil.checkEquals(another, cp);
assertEquals("v", cp.getProperties().get("k"));
assertThat(cp.getProperties().get("k")).isEqualTo("v");
}

private void checkStatistics(int inputStat, int expectStat) throws Exception {
@@ -327,10 +324,10 @@ private void checkStatistics(int inputStat, int expectStat) throws Exception {
catalog.createTable(path1, catalogTable, false);

CatalogTableStatistics statistics = catalog.getTableStatistics(path1);
assertEquals(expectStat, statistics.getRowCount());
assertEquals(expectStat, statistics.getFileCount());
assertEquals(expectStat, statistics.getRawDataSize());
assertEquals(expectStat, statistics.getTotalSize());
assertThat(statistics.getRowCount()).isEqualTo(expectStat);
assertThat(statistics.getFileCount()).isEqualTo(expectStat);
assertThat(statistics.getRawDataSize()).isEqualTo(expectStat);
assertThat(statistics.getTotalSize()).isEqualTo(expectStat);
}

// ------ utils ------

0 comments on commit a4fa285

Please sign in to comment.