Skip to content
Permalink
Browse files
HIVE-26282: Improve iceberg CTAS error message for unsupported types (#…
…3337) (Laszlo Pinter, reviewed by Peter Vary)
  • Loading branch information
lcspinter committed Jun 3, 2022
1 parent ebaba14 commit 8ec1d4c989baa0c4863084297b73e80e5929044d
Showing 2 changed files with 61 additions and 2 deletions.
@@ -83,7 +83,8 @@ Type convertType(TypeInfo typeInfo) {
return Types.BooleanType.get();
case BYTE:
case SHORT:
Preconditions.checkArgument(autoConvert, "Unsupported Hive type: %s, use integer instead",
Preconditions.checkArgument(autoConvert, "Unsupported Hive type: %s, use integer " +
"instead or enable automatic type conversion, set 'iceberg.mr.schema.auto.conversion' to true",
((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory());

LOG.debug("Using auto conversion from SHORT/BYTE to INTEGER");
@@ -96,7 +97,8 @@ Type convertType(TypeInfo typeInfo) {
return Types.BinaryType.get();
case CHAR:
case VARCHAR:
Preconditions.checkArgument(autoConvert, "Unsupported Hive type: %s, use string instead",
Preconditions.checkArgument(autoConvert, "Unsupported Hive type: %s, use integer " +
"instead or enable automatic type conversion, set 'iceberg.mr.schema.auto.conversion' to true",
((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory());

LOG.debug("Using auto conversion from CHAR/VARCHAR to STRING");
@@ -22,6 +22,7 @@
import java.io.IOException;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.iceberg.AssertHelpers;
import org.apache.iceberg.PartitionSpec;
@@ -32,8 +33,10 @@
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.data.Record;
import org.apache.iceberg.exceptions.NoSuchTableException;
import org.apache.iceberg.mr.InputFormatConfig;
import org.apache.iceberg.mr.TestHelper;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
import org.apache.thrift.TException;
import org.junit.Assert;
@@ -222,4 +225,58 @@ public void testCTASFollowedByTruncate() throws IOException {
objects = shell.executeStatement("SELECT * FROM target");
Assert.assertTrue(objects.isEmpty());
}

@Test
public void testCTASUnsupportedTypeWithoutAutoConversion() {
Assume.assumeTrue(HiveIcebergSerDe.CTAS_EXCEPTION_MSG, testTableType == TestTables.TestTableType.HIVE_CATALOG);
Map<String, Type> notSupportedTypes = ImmutableMap.of(
"TINYINT", Types.IntegerType.get(),
"SMALLINT", Types.IntegerType.get(),
"VARCHAR(1)", Types.StringType.get(),
"CHAR(1)", Types.StringType.get());


for (String notSupportedType : notSupportedTypes.keySet()) {
shell.executeStatement(String.format("CREATE TABLE source (s %s) STORED AS ORC", notSupportedType));
AssertHelpers.assertThrows("should throw exception", IllegalArgumentException.class,
"Unsupported Hive type: ", () -> {
shell.executeStatement(String.format(
"CREATE TABLE target STORED BY ICEBERG %s %s AS SELECT * FROM source",
testTables.locationForCreateTableSQL(TableIdentifier.of("default", "target")),
testTables.propertiesForCreateTableSQL(
ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.toString())
)
));
});
shell.executeStatement("DROP TABLE source");
}
}

@Test
public void testCTASUnsupportedTypeWithAutoConversion() {
Assume.assumeTrue(HiveIcebergSerDe.CTAS_EXCEPTION_MSG, testTableType == TestTables.TestTableType.HIVE_CATALOG);
Map<String, Type> notSupportedTypes = ImmutableMap.of(
"TINYINT", Types.IntegerType.get(),
"SMALLINT", Types.IntegerType.get(),
"VARCHAR(1)", Types.StringType.get(),
"CHAR(1)", Types.StringType.get());

shell.setHiveSessionValue(InputFormatConfig.SCHEMA_AUTO_CONVERSION, "true");

for (String notSupportedType : notSupportedTypes.keySet()) {
shell.executeStatement(String.format("CREATE TABLE source (s %s) STORED AS ORC", notSupportedType));
shell.executeStatement(String.format(
"CREATE TABLE target STORED BY ICEBERG %s %s AS SELECT * FROM source",
testTables.locationForCreateTableSQL(TableIdentifier.of("default", "target")),
testTables.propertiesForCreateTableSQL(
ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.toString())
)
));

org.apache.iceberg.Table icebergTable = testTables.loadTable(TableIdentifier.of("default", "target"));
Assert.assertEquals(notSupportedTypes.get(notSupportedType), icebergTable.schema().columns().get(0).type());
shell.executeStatement("DROP TABLE source");
shell.executeStatement("DROP TABLE target");
}
}
}

0 comments on commit 8ec1d4c

Please sign in to comment.