Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] Add support to add and show (create table) extra hive table properties issue 954 #1176

Open
wants to merge 1 commit into
base: master
from
Open
Changes from all commits
Commits
File filter...
Filter file types
Jump to…
Jump to file or symbol
Failed to load files and symbols.

Always

Just for now

@@ -118,6 +118,7 @@
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static com.google.common.collect.Iterables.concat;
import static com.google.common.collect.Streams.stream;
import static io.airlift.json.JsonCodec.mapJsonCodec;
import static io.prestosql.plugin.hive.HiveAnalyzeProperties.getPartitionList;
import static io.prestosql.plugin.hive.HiveBasicStatistics.createEmptyStatistics;
import static io.prestosql.plugin.hive.HiveBasicStatistics.createZeroStatistics;
@@ -152,6 +153,7 @@
import static io.prestosql.plugin.hive.HiveTableProperties.CSV_QUOTE;
import static io.prestosql.plugin.hive.HiveTableProperties.CSV_SEPARATOR;
import static io.prestosql.plugin.hive.HiveTableProperties.EXTERNAL_LOCATION_PROPERTY;
import static io.prestosql.plugin.hive.HiveTableProperties.EXTRA_PROPERTIES;
import static io.prestosql.plugin.hive.HiveTableProperties.ORC_BLOOM_FILTER_COLUMNS;
import static io.prestosql.plugin.hive.HiveTableProperties.ORC_BLOOM_FILTER_FPP;
import static io.prestosql.plugin.hive.HiveTableProperties.PARTITIONED_BY_PROPERTY;
@@ -163,6 +165,7 @@
import static io.prestosql.plugin.hive.HiveTableProperties.getBucketProperty;
import static io.prestosql.plugin.hive.HiveTableProperties.getCsvProperty;
import static io.prestosql.plugin.hive.HiveTableProperties.getExternalLocation;
import static io.prestosql.plugin.hive.HiveTableProperties.getExtraProperties;
import static io.prestosql.plugin.hive.HiveTableProperties.getHiveStorageFormat;
import static io.prestosql.plugin.hive.HiveTableProperties.getOrcBloomFilterColumns;
import static io.prestosql.plugin.hive.HiveTableProperties.getOrcBloomFilterFpp;
@@ -533,6 +536,12 @@ private ConnectorTableMetadata doGetTableMetadata(SchemaTableName tableName)

Optional<String> comment = Optional.ofNullable(table.get().getParameters().get(TABLE_COMMENT));

String extraProperties = table.get().getParameters().get(EXTRA_PROPERTIES);
if (extraProperties != null) {
properties.put(EXTRA_PROPERTIES, mapJsonCodec(String.class, String.class).fromJson(extraProperties));
}


return new ConnectorTableMetadata(tableName, columns.build(), properties.build(), comment);
}

@@ -819,6 +828,12 @@ public void createTable(ConnectorSession session, ConnectorTableMetadata tableMe
tableProperties.put(CSV_SEPARATOR_KEY, separator.toString());
});

//Extra properties

This comment has been minimized.

Copy link
@dain

dain Aug 1, 2019

Member

nit: add a space after the //

Map<String, String> extraProperties = getExtraProperties(tableMetadata.getProperties());
if (extraProperties != null) {
tableProperties.put(EXTRA_PROPERTIES, mapJsonCodec(String.class, String.class).toJson(extraProperties));
}

// Table comment property
tableMetadata.getComment().ifPresent(value -> tableProperties.put(TABLE_COMMENT, value));

@@ -22,6 +22,7 @@
import javax.inject.Inject;

import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -55,6 +56,8 @@
public static final String CSV_SEPARATOR = "csv_separator";
public static final String CSV_QUOTE = "csv_quote";
public static final String CSV_ESCAPE = "csv_escape";
public static final String EXTRA_PROPERTIES = "extra_properties";


private final List<PropertyMetadata<?>> tableProperties;

@@ -133,7 +136,16 @@ public HiveTableProperties(TypeManager typeManager, HiveConfig config)
integerProperty(TEXTFILE_SKIP_FOOTER_LINE_COUNT, "Number of footer lines", null, false),
stringProperty(CSV_SEPARATOR, "CSV separator character", null, false),
stringProperty(CSV_QUOTE, "CSV quote character", null, false),
stringProperty(CSV_ESCAPE, "CSV escape character", null, false));
stringProperty(CSV_ESCAPE, "CSV escape character", null, false),
new PropertyMetadata<>(
EXTRA_PROPERTIES,
"asdfafsd",
typeManager.getType(parseTypeSignature("map(varchar, varchar)")),
Map.class,
new HashMap(),
false,
value -> ((Map<String, String>) value),
value -> value));
}

public List<PropertyMetadata<?>> getTableProperties()
@@ -247,4 +259,9 @@ private static String sortingColumnToString(SortingColumn column)
{
return column.getColumnName() + ((column.getOrder() == DESCENDING) ? " DESC" : "");
}

public static Map<String, String> getExtraProperties(Map<String, Object> tableProperties)
{
return (Map<String, String>) tableProperties.get(EXTRA_PROPERTIES);
}
}
@@ -411,6 +411,10 @@ private static Expression toExpression(Object value)
.collect(toList()));
}

if (value instanceof Map) {
//TODO: How to convert map to an expression here

This comment has been minimized.

Copy link
@ankitdixit

ankitdixit Jul 25, 2019

Author Member

There does not seem to exist a map construct which can directly be converted to expression, please suggest what to do here

This comment has been minimized.

Copy link
@dain

dain Aug 1, 2019

Member

I think invoking map_from_entries is the cleanest.

if (value instanceof Map) {
    Map<?, ?> map = (Map<?, ?>) value;
    return new FunctionCall(QualifiedName.of("map_from_entries"), ImmutableList.of(new ArrayConstructor(map.entrySet().stream()
            .map(entry -> new Row(asList(toExpression(entry.getKey()), toExpression(entry.getValue()))))
            .collect(toList()))));
}

This comment has been minimized.

Copy link
@findepi

findepi Aug 2, 2019

Member

toImmutableList 🙂

}

throw new PrestoException(INVALID_TABLE_PROPERTY, format("Failed to convert object of type %s to expression: %s", value.getClass().getName(), value));
}

ProTip! Use n and p to navigate between commits in a pull request.
You can’t perform that action at this time.