diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java index d7891fe76017..330f67e32bb2 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java @@ -22,7 +22,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; @@ -31,11 +30,8 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.iceberg.Schema; import org.apache.iceberg.expressions.Expressions; -import org.apache.iceberg.expressions.Literal; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; -import org.apache.iceberg.relocated.com.google.common.base.Splitter; import org.apache.iceberg.relocated.com.google.common.collect.Lists; -import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.slf4j.Logger; @@ -63,9 +59,9 @@ static Schema convert(List names, List typeInfos, List return new Schema(converter.convertInternal(names, typeInfos, defaultValues, comments)); } - static Type convert(TypeInfo typeInfo, boolean autoConvert) { + public static Type convert(TypeInfo typeInfo, boolean autoConvert, String defaultValue) { HiveSchemaConverter converter = new HiveSchemaConverter(autoConvert); - return converter.convertType(typeInfo, null); + return converter.convertType(typeInfo, defaultValue); } List convertInternal(List names, List typeInfos, @@ -86,7 +82,7 @@ List convertInternal(List names, List typeI if (defaultValues.containsKey(columnName)) { if (type.isPrimitiveType()) { - Object icebergDefaultValue = getDefaultValue(defaultValues.get(columnName), type); + Object icebergDefaultValue = HiveSchemaUtil.getDefaultValue(defaultValues.get(columnName), type); fieldBuilder.withWriteDefault(Expressions.lit(icebergDefaultValue)); } else if (!type.isStructType()) { throw new UnsupportedOperationException( @@ -99,13 +95,6 @@ List convertInternal(List names, List typeI return result; } - private static Object getDefaultValue(String defaultValue, Type type) { - return switch (type.typeId()) { - case DATE, TIME, TIMESTAMP, TIMESTAMP_NANO -> Literal.of(stripQuotes(defaultValue)).to(type).value(); - default -> Conversions.fromPartitionString(type, stripQuotes(defaultValue)); - }; - } - Type convertType(TypeInfo typeInfo, String defaultValue) { switch (typeInfo.getCategory()) { case PRIMITIVE: @@ -162,7 +151,7 @@ Type convertType(TypeInfo typeInfo, String defaultValue) { StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; List fields = convertInternal(structTypeInfo.getAllStructFieldNames(), structTypeInfo.getAllStructFieldTypeInfos(), - getDefaultValuesMap(defaultValue), Collections.emptyList()); + HiveSchemaUtil.getDefaultValuesMap(defaultValue), Collections.emptyList()); return Types.StructType.of(fields); case MAP: MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo; @@ -182,20 +171,4 @@ Type convertType(TypeInfo typeInfo, String defaultValue) { throw new IllegalArgumentException("Unknown type " + typeInfo.getCategory()); } } - - private static Map getDefaultValuesMap(String defaultValue) { - if (StringUtils.isEmpty(defaultValue)) { - return Collections.emptyMap(); - } - // For Struct, the default value is expected to be in key:value format - return Splitter.on(',').trimResults().withKeyValueSeparator(':').split(stripQuotes(defaultValue)); - } - - public static String stripQuotes(String val) { - if (val.charAt(0) == '\'' && val.charAt(val.length() - 1) == '\'' || - val.charAt(0) == '"' && val.charAt(val.length() - 1) == '"') { - return val.substring(1, val.length() - 1); - } - return val; - } } diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java index ce563b1e55d0..362ea6a10064 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java @@ -27,6 +27,7 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; @@ -34,8 +35,11 @@ import org.apache.iceberg.Schema; import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; +import org.apache.iceberg.expressions.Literal; +import org.apache.iceberg.relocated.com.google.common.base.Splitter; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.DateTimeUtil; @@ -140,23 +144,29 @@ public static TypeInfo convert(Type type) { /** * Converts a Hive typeInfo object to an Iceberg type. - * @param typeInfo The Hive type + * + * @param typeInfo The Hive type + * @param defaultValue the default value for the column, if any * @return The Iceberg type */ - public static Type convert(TypeInfo typeInfo) { - return HiveSchemaConverter.convert(typeInfo, false); + public static Type convert(TypeInfo typeInfo, String defaultValue) { + return HiveSchemaConverter.convert(typeInfo, false, defaultValue); } /** * Returns a SchemaDifference containing those fields which are present in only one of the collections, as well as * those fields which are present in both (in terms of the name) but their type or comment has changed. - * @param minuendCollection Collection of fields to subtract from + * + * @param minuendCollection Collection of fields to subtract from * @param subtrahendCollection Collection of fields to subtract - * @param bothDirections Whether or not to compute the missing fields from the minuendCollection as well + * @param schema the iceberg table schema, if available. Used to compare default values + * @param defaultValues the column default values + * @param bothDirections Whether or not to compute the missing fields from the minuendCollection as well * @return the difference between the two schemas */ public static SchemaDifference getSchemaDiff(Collection minuendCollection, - Collection subtrahendCollection, boolean bothDirections) { + Collection subtrahendCollection, Schema schema, Map defaultValues, + boolean bothDirections) { SchemaDifference difference = new SchemaDifference(); for (FieldSchema first : minuendCollection) { @@ -178,13 +188,61 @@ public static SchemaDifference getSchemaDiff(Collection minuendColl } if (bothDirections) { - SchemaDifference otherWay = getSchemaDiff(subtrahendCollection, minuendCollection, false); + SchemaDifference otherWay = getSchemaDiff(subtrahendCollection, minuendCollection, null, defaultValues, false); otherWay.getMissingFromSecond().forEach(difference::addMissingFromFirst); } + if (schema != null) { + for (Types.NestedField field : schema.columns()) { + if (!isRemovedField(field, difference.getMissingFromFirst())) { + getDefaultValDiff(field, defaultValues, difference); + } + } + } + return difference; } + private static boolean isRemovedField(Types.NestedField field, List missingFields) { + for (FieldSchema fieldSchema : missingFields) { + if (fieldSchema.getName().equalsIgnoreCase(field.name())) { + return true; + } + } + return false; + } + + /** + * Computes whether the default value has changed for the given field. + * @param field the field to check for default value change + * @param defaultValues the default values for the table schema, if available. Used to compare default values + * @param difference the SchemaDifference object to update with the default value change if any + */ + private static void getDefaultValDiff(Types.NestedField field, Map defaultValues, + SchemaDifference difference) { + + String defaultStr = defaultValues.get(field.name()); + + // Skip if no default at all + if (defaultStr == null && field.writeDefault() == null) { + return; + } + + if (field.type().isPrimitiveType()) { + Object expectedDefault = HiveSchemaUtil.getDefaultValue(defaultStr, field.type()); + if (!Objects.equals(expectedDefault, field.writeDefault())) { + difference.addDefaultChanged(field, expectedDefault); + } + } else if (field.type().isStructType()) { + Map structDefaults = getDefaultValuesMap(defaultStr); + + for (Types.NestedField nested : field.type().asStructType().fields()) { + getDefaultValDiff(nested, structDefaults, difference); + } + } + } + + /** * Compares two lists of columns to each other to find the (singular) column that was moved. This works ideally for * identifying the column that was moved by an ALTER TABLE ... CHANGE COLUMN command. @@ -248,6 +306,7 @@ public static class SchemaDifference { private final List missingFromSecond = Lists.newArrayList(); private final List typeChanged = Lists.newArrayList(); private final List commentChanged = Lists.newArrayList(); + private final Map defaultChanged = Maps.newHashMap(); public List getMissingFromFirst() { return missingFromFirst; @@ -265,9 +324,13 @@ public List getCommentChanged() { return commentChanged; } + public Map getDefaultChanged() { + return defaultChanged; + } + public boolean isEmpty() { return missingFromFirst.isEmpty() && missingFromSecond.isEmpty() && typeChanged.isEmpty() && - commentChanged.isEmpty(); + commentChanged.isEmpty() && defaultChanged.isEmpty(); } void addMissingFromFirst(FieldSchema field) { @@ -285,6 +348,10 @@ void addTypeChanged(FieldSchema field) { void addCommentChanged(FieldSchema field) { commentChanged.add(field); } + + void addDefaultChanged(Types.NestedField field, Object defaultValue) { + defaultChanged.put(field, defaultValue); + } } @@ -408,4 +475,35 @@ public static Object convertToWriteType(Object value, Type type) { return value; // fallback } + + public static Map getDefaultValuesMap(String defaultValue) { + if (StringUtils.isEmpty(defaultValue)) { + return Collections.emptyMap(); + } + // For Struct, the default value is expected to be in key:value format + return Splitter.on(',').trimResults().withKeyValueSeparator(':').split(stripQuotes(defaultValue)); + } + + public static String stripQuotes(String val) { + if (val.charAt(0) == '\'' && val.charAt(val.length() - 1) == '\'' || + val.charAt(0) == '"' && val.charAt(val.length() - 1) == '"') { + return val.substring(1, val.length() - 1); + } + return val; + } + + public static Object getDefaultValue(String defaultValue, Type type) { + if (defaultValue == null) { + return null; + } + return switch (type.typeId()) { + case DATE, TIME, TIMESTAMP, TIMESTAMP_NANO -> + Literal.of(stripQuotes(defaultValue)).to(type).value(); + default -> Conversions.fromPartitionString(type, stripQuotes(defaultValue)); + }; + } + + public static Type getStructType(TypeInfo typeInfo, String defaultValue) { + return HiveSchemaConverter.convert(typeInfo, false, defaultValue); + } } diff --git a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java index 1b2bae823ce6..6daf3aeca5d7 100644 --- a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java +++ b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java @@ -220,7 +220,7 @@ private void checkConvert(TypeInfo typeInfo, Type type) { // Convert to TypeInfo assertThat(HiveSchemaUtil.convert(type)).isEqualTo(typeInfo); // Convert to Type - assertEquals(type, HiveSchemaUtil.convert(typeInfo)); + assertEquals(type, HiveSchemaUtil.convert(typeInfo, null)); } /** diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java index 6c1a696f5e26..77d9915adde4 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java @@ -31,6 +31,7 @@ import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.hadoop.conf.Configuration; @@ -45,6 +46,7 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.RequestPartsSpec; +import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; @@ -102,6 +104,7 @@ import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.expressions.Literal; import org.apache.iceberg.expressions.ResidualEvaluator; import org.apache.iceberg.expressions.UnboundPredicate; import org.apache.iceberg.expressions.UnboundTerm; @@ -163,6 +166,7 @@ public class HiveIcebergMetaHook extends BaseHiveIcebergMetaHook { private Transaction transaction; private AlterTableType currentAlterTableOp; private HiveLock commitLock; + private List sqlDefaultConstraints; public HiveIcebergMetaHook(Configuration conf) { super(conf); @@ -502,6 +506,7 @@ public void commitAlterTable(org.apache.hadoop.hive.metastore.api.Table hmsTable if (transaction != null) { transaction.commitTransaction(); } + setSqlDefaultConstraints(); break; case ADDPROPS: case DROPPROPS: @@ -518,6 +523,16 @@ public void commitAlterTable(org.apache.hadoop.hive.metastore.api.Table hmsTable } } + private void setSqlDefaultConstraints() { + try { + if (sqlDefaultConstraints != null && !sqlDefaultConstraints.isEmpty()) { + SessionState.get().getHiveDb().addDefaultConstraint(sqlDefaultConstraints); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + @Override public void rollbackAlterTable(org.apache.hadoop.hive.metastore.api.Table hmsTable, EnvironmentContext context) { if (commitLock == null) { @@ -654,15 +669,24 @@ private void setFileFormat(String format) { private void handleAddColumns(org.apache.hadoop.hive.metastore.api.Table hmsTable) { Collection addedCols = - HiveSchemaUtil.getSchemaDiff(hmsTable.getSd().getCols(), HiveSchemaUtil.convert(icebergTable.schema()), false) + HiveSchemaUtil.getSchemaDiff(hmsTable.getSd().getCols(), HiveSchemaUtil.convert(icebergTable.schema()), null, + null, false) .getMissingFromSecond(); if (!addedCols.isEmpty()) { transaction = icebergTable.newTransaction(); updateSchema = transaction.updateSchema(); } + this.sqlDefaultConstraints = + (List) SessionStateUtil.getResource(conf, SessionStateUtil.COLUMN_DEFAULTS).orElse(null); + Map defaultValues = Stream.ofNullable(sqlDefaultConstraints).flatMap(Collection::stream) + .collect(Collectors.toMap(SQLDefaultConstraint::getColumn_name, SQLDefaultConstraint::getDefault_value)); for (FieldSchema addedCol : addedCols) { - updateSchema.addColumn(addedCol.getName(), - HiveSchemaUtil.convert(TypeInfoUtils.getTypeInfoFromTypeString(addedCol.getType())), addedCol.getComment()); + String defaultValue = defaultValues.get(addedCol.getName()); + Type type = HiveSchemaUtil.convert(TypeInfoUtils.getTypeInfoFromTypeString(addedCol.getType()), defaultValue); + Literal defaultVal = Optional.ofNullable(defaultValue).filter(v -> !type.isStructType()) + .map(v -> Expressions.lit(HiveSchemaUtil.getDefaultValue(v, type))).orElse(null); + + updateSchema.addColumn(addedCol.getName(), type, addedCol.getComment(), defaultVal); } updateSchema.commit(); } @@ -671,7 +695,8 @@ private void handleDropColumn(org.apache.hadoop.hive.metastore.api.Table hmsTabl List hmsCols = hmsTable.getSd().getCols(); List icebergCols = HiveSchemaUtil.convert(icebergTable.schema()); - List removedCols = HiveSchemaUtil.getSchemaDiff(icebergCols, hmsCols, false).getMissingFromSecond(); + List removedCols = + HiveSchemaUtil.getSchemaDiff(icebergCols, hmsCols, null, null, false).getMissingFromSecond(); if (removedCols.isEmpty()) { return; } @@ -685,7 +710,14 @@ private void handleDropColumn(org.apache.hadoop.hive.metastore.api.Table hmsTabl private void handleReplaceColumns(org.apache.hadoop.hive.metastore.api.Table hmsTable) throws MetaException { List hmsCols = hmsTable.getSd().getCols(); List icebergCols = HiveSchemaUtil.convert(icebergTable.schema()); - HiveSchemaUtil.SchemaDifference schemaDifference = HiveSchemaUtil.getSchemaDiff(hmsCols, icebergCols, true); + + List defaultConstraints = + (List) SessionStateUtil.getResource(conf, SessionStateUtil.COLUMN_DEFAULTS).orElse(null); + Map defaultValues = Optional.ofNullable(defaultConstraints).orElse(Collections.emptyList()).stream() + .collect(Collectors.toMap(SQLDefaultConstraint::getColumn_name, SQLDefaultConstraint::getDefault_value)); + + HiveSchemaUtil.SchemaDifference schemaDifference = + HiveSchemaUtil.getSchemaDiff(hmsCols, icebergCols, icebergTable.schema(), defaultValues, true); // if there are columns dropped, let's remove them from the iceberg schema as well so we can compare the order if (!schemaDifference.getMissingFromFirst().isEmpty()) { @@ -697,9 +729,11 @@ private void handleReplaceColumns(org.apache.hadoop.hive.metastore.api.Table hms // limit the scope of this operation to only dropping columns if (!schemaDifference.getMissingFromSecond().isEmpty() || !schemaDifference.getTypeChanged().isEmpty() || - !schemaDifference.getCommentChanged().isEmpty() || outOfOrder != null) { + !schemaDifference.getCommentChanged().isEmpty() || outOfOrder != null || + !schemaDifference.getDefaultChanged().isEmpty()) { throw new MetaException("Unsupported operation to use REPLACE COLUMNS for adding a column, changing a " + - "column type, column comment or reordering columns. Only use REPLACE COLUMNS for dropping columns. " + + "column type, column comment, column default or reordering columns. " + + "Only use REPLACE COLUMNS for dropping columns. " + "For the other operations, consider using the ADD COLUMNS or CHANGE COLUMN commands."); } @@ -722,7 +756,12 @@ private void handleChangeColumn(org.apache.hadoop.hive.metastore.api.Table hmsTa List hmsCols = hmsTable.getSd().getCols(); List icebergCols = HiveSchemaUtil.convert(icebergTable.schema()); // compute schema difference for renames, type/comment changes - HiveSchemaUtil.SchemaDifference schemaDifference = HiveSchemaUtil.getSchemaDiff(hmsCols, icebergCols, true); + List defaultConstraints = + (List) SessionStateUtil.getResource(conf, SessionStateUtil.COLUMN_DEFAULTS).orElse(null); + Map defaultValues = Optional.ofNullable(defaultConstraints).orElse(Collections.emptyList()).stream() + .collect(Collectors.toMap(SQLDefaultConstraint::getColumn_name, SQLDefaultConstraint::getDefault_value)); + HiveSchemaUtil.SchemaDifference schemaDifference = + HiveSchemaUtil.getSchemaDiff(hmsCols, icebergCols, null, null, true); // check column reorder (which could happen even in the absence of any rename, type or comment change) Map renameMapping = ImmutableMap.of(); if (!schemaDifference.getMissingFromSecond().isEmpty()) { @@ -732,12 +771,12 @@ private void handleChangeColumn(org.apache.hadoop.hive.metastore.api.Table hmsTa } Pair> outOfOrder = HiveSchemaUtil.getReorderedColumn(hmsCols, icebergCols, renameMapping); - if (!schemaDifference.isEmpty() || outOfOrder != null) { + if (!schemaDifference.isEmpty() || outOfOrder != null || !defaultValues.isEmpty()) { transaction = icebergTable.newTransaction(); updateSchema = transaction.updateSchema(); } else { // we should get here if the user didn't change anything about the column - // i.e. no changes to the name, type, comment or order + // i.e. no changes to the name, type, comment, default or order LOG.info("Found no difference between new and old schema for ALTER TABLE CHANGE COLUMN for" + " table: {}. There will be no Iceberg commit.", hmsTable.getTableName()); return; @@ -776,11 +815,41 @@ private void handleChangeColumn(org.apache.hadoop.hive.metastore.api.Table hmsTa updateSchema.moveFirst(outOfOrder.first()); } } + + // case 5: handle change of default values + handleDefaultValues(defaultValues, renameMapping, icebergTable.schema().columns(), ""); updateSchema.commit(); handlePartitionRename(schemaDifference); } + /** + * Updates the default values of the fields. + * @param defaultValues the map containing the default values. + * @param renameMapping the rename mapping of the columns + * @param columns the columns of the table + * @param prefix the prefix of the columns, empty unless a filed of struct. + */ + private void handleDefaultValues(Map defaultValues, Map renameMapping, + List columns, String prefix) { + if (!defaultValues.isEmpty()) { + for (Map.Entry field : defaultValues.entrySet()) { + String simpleName = + renameMapping.containsKey(field.getKey()) ? renameMapping.get(field.getKey()) : field.getKey(); + String qualifiedName = prefix + simpleName; + Type fieldType = + columns.stream().filter(col -> col.name().equalsIgnoreCase(simpleName)).findFirst().get().type(); + if (fieldType.isStructType()) { + Map structDefaults = HiveSchemaUtil.getDefaultValuesMap(field.getValue()); + handleDefaultValues(structDefaults, renameMapping, fieldType.asStructType().fields(), qualifiedName + "."); + } else { + updateSchema.updateColumnDefault(qualifiedName, + Expressions.lit(HiveSchemaUtil.getDefaultValue(field.getValue(), fieldType))); + } + } + } + } + private void handlePartitionRename(HiveSchemaUtil.SchemaDifference schemaDifference) { // in case a partition column has been renamed, spec needs to be adjusted too if (!schemaDifference.getMissingFromSecond().isEmpty()) { @@ -795,7 +864,7 @@ private void handlePartitionRename(HiveSchemaUtil.SchemaDifference schemaDiffere } private Type.PrimitiveType getPrimitiveTypeOrThrow(FieldSchema field) throws MetaException { - Type newType = HiveSchemaUtil.convert(TypeInfoUtils.getTypeInfoFromTypeString(field.getType())); + Type newType = HiveSchemaUtil.convert(TypeInfoUtils.getTypeInfoFromTypeString(field.getType()), null); if (!(newType instanceof Type.PrimitiveType)) { throw new MetaException(String.format("Cannot promote type of column: '%s' to a non-primitive type: %s.", field.getName(), newType)); diff --git a/iceberg/iceberg-handler/src/test/queries/negative/iceberg_replace_column_with_default_change.q b/iceberg/iceberg-handler/src/test/queries/negative/iceberg_replace_column_with_default_change.q new file mode 100644 index 000000000000..3eefaf306b00 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/negative/iceberg_replace_column_with_default_change.q @@ -0,0 +1,12 @@ +CREATE TABLE ice_t ( + id INT, + name STRING DEFAULT 'unknown', + age INT DEFAULT 25 +) +STORED BY ICEBERG +TBLPROPERTIES ('format-version'='3'); + +ALTER TABLE ice_t REPLACE COLUMNS ( + id INT, + name STRING DEFAULT 'unknown1', + age INT DEFAULT 25); \ No newline at end of file diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_alter_default_column.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_alter_default_column.q new file mode 100644 index 000000000000..4ec6abf4faf3 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_alter_default_column.q @@ -0,0 +1,49 @@ +CREATE TABLE ice_t ( + id INT) +STORED BY ICEBERG +TBLPROPERTIES ('format-version'='3'); + +INSERT INTO ice_t (id) VALUES (1); + +ALTER TABLE ice_t ADD COLUMNS (point STRUCT DEFAULT 'x:100,y:99', + name STRING DEFAULT 'unknown', + age INT DEFAULT 25, + salary DOUBLE DEFAULT 50000.0, + is_active BOOLEAN DEFAULT TRUE, + created_date DATE DEFAULT '2024-01-01', + created_ts TIMESTAMP DEFAULT '2024-01-01T10:00:00', + score DECIMAL(5,2) DEFAULT 100.00, + category STRING DEFAULT 'general'); + +INSERT INTO ice_t (id) VALUES (2); + +SELECT * FROM ice_t ORDER BY id; + +ALTER TABLE ice_t REPLACE COLUMNS (id INT, + point STRUCT DEFAULT 'x:100,y:99', + name STRING DEFAULT 'unknown', + age INT DEFAULT 25, + salary DOUBLE DEFAULT 50000.0, + is_active BOOLEAN DEFAULT TRUE, + created_date DATE DEFAULT '2024-01-01', + created_ts TIMESTAMP DEFAULT '2024-01-01T10:00:00', + category STRING DEFAULT 'general'); + +SELECT * FROM ice_t ORDER BY id; + +-- change default of a field of Struct column +ALTER TABLE ice_t CHANGE COLUMN point point STRUCT DEFAULT 'x:100,y:88'; + +-- rename and change default value of age column +ALTER TABLE ice_t CHANGE COLUMN age age_new int DEFAULT 21; + +INSERT INTO ice_t (id) VALUES (3); + +SELECT * FROM ice_t ORDER BY id; + +-- Rename the struct column with default changes +ALTER TABLE ice_t CHANGE COLUMN point point_new STRUCT DEFAULT 'x:55,y:88'; + +INSERT INTO ice_t (id) VALUES (4); + +SELECT * FROM ice_t ORDER BY id; \ No newline at end of file diff --git a/iceberg/iceberg-handler/src/test/results/negative/iceberg_replace_column_with_default_change.q.out b/iceberg/iceberg-handler/src/test/results/negative/iceberg_replace_column_with_default_change.q.out new file mode 100644 index 000000000000..0297b5f65ef8 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/negative/iceberg_replace_column_with_default_change.q.out @@ -0,0 +1,30 @@ +PREHOOK: query: CREATE TABLE ice_t ( + id INT, + name STRING DEFAULT 'unknown', + age INT DEFAULT 25 +) +STORED BY ICEBERG +TBLPROPERTIES ('format-version'='3') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_t +POSTHOOK: query: CREATE TABLE ice_t ( + id INT, + name STRING DEFAULT 'unknown', + age INT DEFAULT 25 +) +STORED BY ICEBERG +TBLPROPERTIES ('format-version'='3') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_t +PREHOOK: query: ALTER TABLE ice_t REPLACE COLUMNS ( + id INT, + name STRING DEFAULT 'unknown1', + age INT DEFAULT 25) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@ice_t +PREHOOK: Output: default@ice_t +FAILED: Execution Error, return code 40013 from org.apache.hadoop.hive.ql.ddl.DDLTask. Unable to alter table. MetaException(message:Unsupported operation to use REPLACE COLUMNS for adding a column, changing a column type, column comment, column default or reordering columns. Only use REPLACE COLUMNS for dropping columns. For the other operations, consider using the ADD COLUMNS or CHANGE COLUMN commands.) +#### A masked pattern was here #### + diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_alter_default_column.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_alter_default_column.q.out new file mode 100644 index 000000000000..bf7cdcf4d962 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_alter_default_column.q.out @@ -0,0 +1,161 @@ +PREHOOK: query: CREATE TABLE ice_t ( + id INT) +STORED BY ICEBERG +TBLPROPERTIES ('format-version'='3') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_t +POSTHOOK: query: CREATE TABLE ice_t ( + id INT) +STORED BY ICEBERG +TBLPROPERTIES ('format-version'='3') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_t +PREHOOK: query: INSERT INTO ice_t (id) VALUES (1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_t +POSTHOOK: query: INSERT INTO ice_t (id) VALUES (1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_t +PREHOOK: query: ALTER TABLE ice_t ADD COLUMNS (point STRUCT DEFAULT 'x:100,y:99', + name STRING DEFAULT 'unknown', + age INT DEFAULT 25, + salary DOUBLE DEFAULT 50000.0, + is_active BOOLEAN DEFAULT TRUE, + created_date DATE DEFAULT '2024-01-01', + created_ts TIMESTAMP DEFAULT '2024-01-01T10:00:00', + score DECIMAL(5,2) DEFAULT 100.00, + category STRING DEFAULT 'general') +PREHOOK: type: ALTERTABLE_ADDCOLS +PREHOOK: Input: default@ice_t +PREHOOK: Output: default@ice_t +POSTHOOK: query: ALTER TABLE ice_t ADD COLUMNS (point STRUCT DEFAULT 'x:100,y:99', + name STRING DEFAULT 'unknown', + age INT DEFAULT 25, + salary DOUBLE DEFAULT 50000.0, + is_active BOOLEAN DEFAULT TRUE, + created_date DATE DEFAULT '2024-01-01', + created_ts TIMESTAMP DEFAULT '2024-01-01T10:00:00', + score DECIMAL(5,2) DEFAULT 100.00, + category STRING DEFAULT 'general') +POSTHOOK: type: ALTERTABLE_ADDCOLS +POSTHOOK: Input: default@ice_t +POSTHOOK: Output: default@ice_t +PREHOOK: query: INSERT INTO ice_t (id) VALUES (2) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_t +POSTHOOK: query: INSERT INTO ice_t (id) VALUES (2) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_t +PREHOOK: query: SELECT * FROM ice_t ORDER BY id +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_t +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * FROM ice_t ORDER BY id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_t +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 NULL NULL NULL NULL NULL NULL NULL NULL NULL +2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general +PREHOOK: query: ALTER TABLE ice_t REPLACE COLUMNS (id INT, + point STRUCT DEFAULT 'x:100,y:99', + name STRING DEFAULT 'unknown', + age INT DEFAULT 25, + salary DOUBLE DEFAULT 50000.0, + is_active BOOLEAN DEFAULT TRUE, + created_date DATE DEFAULT '2024-01-01', + created_ts TIMESTAMP DEFAULT '2024-01-01T10:00:00', + category STRING DEFAULT 'general') +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@ice_t +PREHOOK: Output: default@ice_t +POSTHOOK: query: ALTER TABLE ice_t REPLACE COLUMNS (id INT, + point STRUCT DEFAULT 'x:100,y:99', + name STRING DEFAULT 'unknown', + age INT DEFAULT 25, + salary DOUBLE DEFAULT 50000.0, + is_active BOOLEAN DEFAULT TRUE, + created_date DATE DEFAULT '2024-01-01', + created_ts TIMESTAMP DEFAULT '2024-01-01T10:00:00', + category STRING DEFAULT 'general') +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: default@ice_t +POSTHOOK: Output: default@ice_t +PREHOOK: query: SELECT * FROM ice_t ORDER BY id +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_t +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * FROM ice_t ORDER BY id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_t +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 NULL NULL NULL NULL NULL NULL NULL NULL +2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 general +PREHOOK: query: ALTER TABLE ice_t CHANGE COLUMN point point STRUCT DEFAULT 'x:100,y:88' +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: default@ice_t +PREHOOK: Output: default@ice_t +POSTHOOK: query: ALTER TABLE ice_t CHANGE COLUMN point point STRUCT DEFAULT 'x:100,y:88' +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: default@ice_t +POSTHOOK: Output: default@ice_t +PREHOOK: query: ALTER TABLE ice_t CHANGE COLUMN age age_new int DEFAULT 21 +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: default@ice_t +PREHOOK: Output: default@ice_t +POSTHOOK: query: ALTER TABLE ice_t CHANGE COLUMN age age_new int DEFAULT 21 +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: default@ice_t +POSTHOOK: Output: default@ice_t +PREHOOK: query: INSERT INTO ice_t (id) VALUES (3) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_t +POSTHOOK: query: INSERT INTO ice_t (id) VALUES (3) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_t +PREHOOK: query: SELECT * FROM ice_t ORDER BY id +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_t +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * FROM ice_t ORDER BY id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_t +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 NULL NULL NULL NULL NULL NULL NULL NULL +2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 general +3 {"x":100,"y":88} unknown 21 50000.0 true 2024-01-01 2024-01-01 10:00:00 general +PREHOOK: query: ALTER TABLE ice_t CHANGE COLUMN point point_new STRUCT DEFAULT 'x:55,y:88' +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: default@ice_t +PREHOOK: Output: default@ice_t +POSTHOOK: query: ALTER TABLE ice_t CHANGE COLUMN point point_new STRUCT DEFAULT 'x:55,y:88' +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: default@ice_t +POSTHOOK: Output: default@ice_t +PREHOOK: query: INSERT INTO ice_t (id) VALUES (4) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_t +POSTHOOK: query: INSERT INTO ice_t (id) VALUES (4) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_t +PREHOOK: query: SELECT * FROM ice_t ORDER BY id +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_t +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * FROM ice_t ORDER BY id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_t +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 NULL NULL NULL NULL NULL NULL NULL NULL +2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 general +3 {"x":100,"y":88} unknown 21 50000.0 true 2024-01-01 2024-01-01 10:00:00 general +4 {"x":55,"y":88} unknown 21 50000.0 true 2024-01-01 2024-01-01 10:00:00 general diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index ddce6aa85af6..8d4e943052ef 100644 --- a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -2286,12 +2286,17 @@ columnRefOrder columnNameType @init { pushMsg("column specification", state); } @after { popMsg(state); } - : colName=identifier colType (KW_COMMENT comment=StringLiteral)? - -> {containExcludedCharForCreateTableColumnName($colName.text)}? {throwColumnNameException()} - -> {$comment == null}? ^(TOK_TABCOL $colName colType) - -> ^(TOK_TABCOL $colName colType $comment) + : colName=identifier colType + (KW_COMMENT comment=StringLiteral)? + defaultClause + -> ^(TOK_TABCOL $colName colType $comment? defaultClause?) ; +defaultClause + : (KW_DEFAULT v=expression -> ^(TOK_DEFAULT_VALUE $v))? + ; + + columnNameTypeOrConstraint @init { pushMsg("column name or constraint", state); } @after { popMsg(state); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/DDLUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/DDLUtils.java index 3ee8d74cea9d..c9b74eca9d5e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/DDLUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/DDLUtils.java @@ -24,6 +24,7 @@ import java.util.Optional; import java.util.Set; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; @@ -31,7 +32,9 @@ import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; +import org.apache.hadoop.hive.ql.ddl.table.constraint.ConstraintsUtils; import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.hooks.Entity.Type; import org.apache.hadoop.hive.ql.metadata.Hive; @@ -248,4 +251,14 @@ public static boolean hasTransformsInPartitionSpec(Table table) { table.getStorageHandler().getPartitionTransformSpec(table).stream() .anyMatch(spec -> spec.getTransformType() != TransformSpec.TransformType.IDENTITY); } + + public static void setDefaultColumnValues(Table table, TableName tableName, + List defaultConstraintsInfo, Configuration conf) throws SemanticException { + boolean isNativeColumnDefaultSupported = table.getStorageHandler() != null && table.getStorageHandler() + .supportsDefaultColumnValues(table.getParameters()); + List defaultConstraints = new ArrayList<>(); + ConstraintsUtils.constraintInfosToDefaultConstraints(tableName, defaultConstraintsInfo, defaultConstraints, + isNativeColumnDefaultSupported); + SessionStateUtil.addResourceOrThrow(conf, SessionStateUtil.COLUMN_DEFAULTS, defaultConstraints); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/add/AlterTableAddColumnsAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/add/AlterTableAddColumnsAnalyzer.java index 4b490132165d..c4d6a75876ed 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/add/AlterTableAddColumnsAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/add/AlterTableAddColumnsAnalyzer.java @@ -18,15 +18,18 @@ package org.apache.hadoop.hive.ql.ddl.table.column.add; +import java.util.ArrayList; import java.util.List; import java.util.Map; import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.QueryState; +import org.apache.hadoop.hive.ql.ddl.DDLUtils; import org.apache.hadoop.hive.ql.ddl.DDLWork; import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory.DDLType; import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableAnalyzer; +import org.apache.hadoop.hive.ql.ddl.table.constraint.ConstraintsUtils; import org.apache.hadoop.hive.ql.exec.TaskFactory; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.metadata.Table; @@ -46,7 +49,10 @@ public AlterTableAddColumnsAnalyzer(QueryState queryState) throws SemanticExcept @Override protected void analyzeCommand(TableName tableName, Map partitionSpec, ASTNode command) throws SemanticException { - List newCols = getColumns((ASTNode) command.getChild(0)); + List defaultConstraintsInfo = new ArrayList<>(); + List newCols = + getColumns((ASTNode) command.getChild(0), true, ctx.getTokenRewriteStream(), new ArrayList<>(), new ArrayList<>(), new ArrayList<>(), + new ArrayList<>(), defaultConstraintsInfo, new ArrayList<>(), conf); boolean isCascade = false; if (null != command.getFirstChildWithType(HiveParser.TOK_CASCADE)) { isCascade = true; @@ -59,6 +65,8 @@ protected void analyzeCommand(TableName tableName, Map partition } addInputsOutputsAlterTable(tableName, partitionSpec, desc, desc.getType(), false); + + DDLUtils.setDefaultColumnValues(table, tableName, defaultConstraintsInfo, conf); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), desc))); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/change/AlterTableChangeColumnAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/change/AlterTableChangeColumnAnalyzer.java index cf017895f44f..d3f34cc62c93 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/change/AlterTableChangeColumnAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/change/AlterTableChangeColumnAnalyzer.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hive.metastore.api.SkewedInfo; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryState; +import org.apache.hadoop.hive.ql.ddl.DDLUtils; import org.apache.hadoop.hive.ql.ddl.DDLWork; import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory.DDLType; import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableAnalyzer; @@ -118,6 +119,7 @@ private Constraints getConstraints(TableName tableName, ASTNode command, String List uniqueConstraints = null; List notNullConstraints = null; List defaultConstraints = null; + List defaultConstraintInfo = null; List checkConstraints = null; if (constraintChild != null) { // Process column constraint @@ -128,10 +130,9 @@ private Constraints getConstraints(TableName tableName, ASTNode command, String checkConstraints, (ASTNode) command.getChild(2), this.ctx.getTokenRewriteStream()); break; case HiveParser.TOK_DEFAULT_VALUE: - defaultConstraints = new ArrayList<>(); - ConstraintsUtils.constraintInfosToDefaultConstraints(tableName, + defaultConstraintInfo = ConstraintsUtils.processDefaultConstraints(constraintChild, ImmutableList.of(newColumnName), - (ASTNode) command.getChild(2), this.ctx.getTokenRewriteStream()), defaultConstraints, false); + (ASTNode) command.getChild(2), this.ctx.getTokenRewriteStream()); break; case HiveParser.TOK_NOT_NULL: notNullConstraints = new ArrayList<>(); @@ -162,8 +163,18 @@ private Constraints getConstraints(TableName tableName, ASTNode command, String ConstraintsUtils.validateCheckConstraint(table.getCols(), checkConstraints, ctx.getConf()); } - if (table.getTableType() == TableType.EXTERNAL_TABLE && - ConstraintsUtils.hasEnabledOrValidatedConstraints(notNullConstraints, defaultConstraints, checkConstraints)) { + boolean isNativeColumnDefaultSupported = table.getStorageHandler() != null && table.getStorageHandler() + .supportsDefaultColumnValues(table.getParameters()); + + if (defaultConstraintInfo != null) { + defaultConstraints = new ArrayList<>(); + ConstraintsUtils.constraintInfosToDefaultConstraints(tableName, defaultConstraintInfo, defaultConstraints, + isNativeColumnDefaultSupported); + DDLUtils.setDefaultColumnValues(table, tableName, defaultConstraintInfo, ctx.getConf()); + } + + if (table.getTableType() == TableType.EXTERNAL_TABLE && ConstraintsUtils.hasEnabledOrValidatedConstraints( + notNullConstraints, defaultConstraints, checkConstraints, isNativeColumnDefaultSupported)) { throw new SemanticException(ErrorMsg.INVALID_CSTR_SYNTAX.getMsg( "Constraints are disallowed with External tables. Only RELY is allowed.")); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/replace/AlterTableReplaceColumnsAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/replace/AlterTableReplaceColumnsAnalyzer.java index 933b80e53931..99a519c2073d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/replace/AlterTableReplaceColumnsAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/replace/AlterTableReplaceColumnsAnalyzer.java @@ -18,15 +18,18 @@ package org.apache.hadoop.hive.ql.ddl.table.column.replace; +import java.util.ArrayList; import java.util.List; import java.util.Map; import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.QueryState; +import org.apache.hadoop.hive.ql.ddl.DDLUtils; import org.apache.hadoop.hive.ql.ddl.DDLWork; import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory.DDLType; import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableAnalyzer; +import org.apache.hadoop.hive.ql.ddl.table.constraint.ConstraintsUtils; import org.apache.hadoop.hive.ql.exec.TaskFactory; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.metadata.Table; @@ -46,7 +49,10 @@ public AlterTableReplaceColumnsAnalyzer(QueryState queryState) throws SemanticEx @Override protected void analyzeCommand(TableName tableName, Map partitionSpec, ASTNode command) throws SemanticException { - List newCols = getColumns((ASTNode) command.getChild(0)); + List defaultConstraintsInfo = new ArrayList<>(); + List newCols = + getColumns((ASTNode) command.getChild(0), true, ctx.getTokenRewriteStream(), new ArrayList<>(), + new ArrayList<>(), new ArrayList<>(), new ArrayList<>(), defaultConstraintsInfo, new ArrayList<>(), conf); boolean isCascade = false; if (null != command.getFirstChildWithType(HiveParser.TOK_CASCADE)) { isCascade = true; @@ -59,6 +65,7 @@ protected void analyzeCommand(TableName tableName, Map partition } addInputsOutputsAlterTable(tableName, partitionSpec, desc, desc.getType(), false); + DDLUtils.setDefaultColumnValues(table, tableName, defaultConstraintsInfo, conf); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), desc))); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/constraint/ConstraintsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/constraint/ConstraintsUtils.java index 0d6950b7f625..daac11c1858c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/constraint/ConstraintsUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/constraint/ConstraintsUtils.java @@ -525,7 +525,8 @@ private static void validateCheckExpr(ExprNodeDesc checkExpr) throws SemanticExc } public static boolean hasEnabledOrValidatedConstraints(List notNullConstraints, - List defaultConstraints, List checkConstraints) { + List defaultConstraints, List checkConstraints, + boolean isNativeColumnDefaultSupported) { if (notNullConstraints != null) { for (SQLNotNullConstraint nnC : notNullConstraints) { if (nnC.isEnable_cstr() || nnC.isValidate_cstr()) { @@ -534,7 +535,7 @@ public static boolean hasEnabledOrValidatedConstraints(List getColumns( ASTNode typeChild = (ASTNode) (child.getChild(1)); col.setType(getTypeStringFromAST(typeChild)); - // child 2 is the optional comment of the column - // child 3 is the optional constraint ASTNode constraintChild = null; if (child.getChildCount() == 4) { col.setComment(unescapeSQLString(child.getChild(2).getText())); @@ -931,8 +928,9 @@ public static List getColumns( constraintChild = (ASTNode) child.getChild(2); } if (constraintChild != null) { - final TableName tName = - getQualifiedTableName((ASTNode) parent.getChild(0), MetaStoreUtils.getDefaultCatalog(conf)); + final TableName tName = constraintChild.getToken().getType() != HiveParser.TOK_DEFAULT_VALUE ? + getQualifiedTableName((ASTNode) parent.getChild(0), MetaStoreUtils.getDefaultCatalog(conf)) : + null; // TODO CAT - for now always use the default catalog. Eventually will want to see if // the user specified a catalog // Process column constraint diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionStateUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionStateUtil.java index 32f38bc0126b..d1367056855b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionStateUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionStateUtil.java @@ -39,6 +39,7 @@ public class SessionStateUtil { private static final String CONFLICT_DETECTION_FILTER = "conflictDetectionFilter."; public static final String DEFAULT_TABLE_LOCATION = "defaultLocation"; public static final String MISSING_COLUMNS = "missingColumns"; + public static final String COLUMN_DEFAULTS = "columnDefaults"; private SessionStateUtil() { }