Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

HIVE-22323 Fix Desc Table bugs #813

Closed
wants to merge 1 commit into from
Closed

HIVE-22323 Fix Desc Table bugs #813

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 2 additions & 1 deletion itests/src/test/resources/testconfiguration.properties
View file
Open in desktop
Expand Up @@ -1026,7 +1026,8 @@ beeline.positive.include=create_merge_compressed.q,\
smb_mapjoin_3.q,\
smb_mapjoin_7.q,\
select_dummy_source.q,\
udf_unix_timestamp.q
udf_unix_timestamp.q, \
desc_table_formatted.q

minimr.query.negative.files=cluster_tasklog_retrieval.q,\
file_with_header_footer_negative.q,\
Expand Down
16 changes: 9 additions & 7 deletions ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/DescTableDesc.java
View file
Open in desktop
Expand Up @@ -19,27 +19,29 @@
package org.apache.hadoop.hive.ql.ddl.table.info;

import java.io.Serializable;
import java.util.List;
import java.util.Map;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.ddl.DDLDesc;
import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.plan.Explain.Level;

import com.google.common.collect.ImmutableList;

/**
* DDL task description for DESC table_name commands.
*/
@Explain(displayName = "Describe Table", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public class DescTableDesc implements DDLDesc, Serializable {
private static final long serialVersionUID = 1L;

private static final String SCHEMA = "col_name,data_type,comment#string:string:string";
private static final String COL_STATS_SCHEMA = "col_name,data_type,min,max,num_nulls," +
"distinct_count,avg_col_len,max_col_len,num_trues,num_falses,bitVector,comment" +
"#string:string:string:string:string:string:string:string:string:string:string:string";
public static String getSchema(boolean colStats) {
return colStats ? COL_STATS_SCHEMA : SCHEMA;
}
public static final String SCHEMA = "col_name,data_type,comment#string:string:string";
public static final String COLUMN_STATISTICS_SCHEMA = "column_property,value#string:string";
public static final List<String> COLUMN_STATISTICS_HEADERS = ImmutableList.of(
"col_name", "data_type", "min", "max", "num_nulls", "distinct_count", "avg_col_len", "max_col_len", "num_trues",
"num_falses", "bit_vector", "comment"
);

private final String resFile;
private final String tableName;
Expand Down
127 changes: 116 additions & 11 deletions ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java
View file
Open in desktop
Expand Up @@ -35,11 +35,13 @@
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.PrincipalType;
Expand Down Expand Up @@ -188,7 +190,7 @@ public void describeTable(DataOutputStream out, String colPath, String tableName
List<FieldSchema> cols, boolean isFormatted, boolean isExt, boolean isOutputPadded,
List<ColumnStatisticsObj> colStats) throws HiveException {
MapBuilder builder = MapBuilder.create();
builder.put("columns", makeColsUnformatted(cols));
builder.put("columns", createColumnsInfo(cols, colStats));

if (isExt) {
if (part != null) {
Expand Down Expand Up @@ -223,20 +225,123 @@ public void describeTable(DataOutputStream out, String colPath, String tableName
asJson(out, builder.build());
}

private List<Map<String, Object>> makeColsUnformatted(List<FieldSchema> cols) {
private List<Map<String, Object>> createColumnsInfo(List<FieldSchema> columns,
List<ColumnStatisticsObj> columnStatisticsList) {
ArrayList<Map<String, Object>> res = new ArrayList<Map<String, Object>>();
for (FieldSchema col : cols) {
res.add(makeOneColUnformatted(col));
for (FieldSchema column : columns) {
ColumnStatisticsData statistics = getStatistics(column, columnStatisticsList);
res.add(createColumnInfo(column, statistics));
}
return res;
}

private Map<String, Object> makeOneColUnformatted(FieldSchema col) {
return MapBuilder.create()
.put("name", col.getName())
.put("type", col.getType())
.put("comment", col.getComment())
private ColumnStatisticsData getStatistics(FieldSchema column, List<ColumnStatisticsObj> columnStatisticsList) {
for (ColumnStatisticsObj columnStatistics : columnStatisticsList) {
if (column.getName().equals(columnStatistics.getColName())) {
return columnStatistics.getStatsData();
}
}

return null;
}

private Map<String, Object> createColumnInfo(FieldSchema column, ColumnStatisticsData statistics) {
Map<String, Object> result = MapBuilder.create()
.put("name", column.getName())
.put("type", column.getType())
.put("comment", column.getComment())
.build();

if (statistics != null) {
if (statistics.isSetBinaryStats()) {
if (statistics.getBinaryStats().isSetNumNulls()) {
result.put("numNulls", statistics.getBinaryStats().getNumNulls());
}
if (statistics.getBinaryStats().isSetAvgColLen()) {
result.put("avgColLen", statistics.getBinaryStats().getAvgColLen());
}
if (statistics.getBinaryStats().isSetMaxColLen()) {
result.put("maxColLen", statistics.getBinaryStats().getMaxColLen());
}
} else if (statistics.isSetStringStats()) {
if (statistics.getStringStats().isSetNumNulls()) {
result.put("numNulls", statistics.getStringStats().getNumNulls());
}
if (statistics.getStringStats().isSetNumDVs()) {
result.put("distinctCount", statistics.getStringStats().getNumDVs());
}
if (statistics.getStringStats().isSetAvgColLen()) {
result.put("avgColLen", statistics.getStringStats().getAvgColLen());
}
if (statistics.getStringStats().isSetMaxColLen()) {
result.put("maxColLen", statistics.getStringStats().getMaxColLen());
}
} else if (statistics.isSetBooleanStats()) {
if (statistics.getBooleanStats().isSetNumNulls()) {
result.put("numNulls", statistics.getBooleanStats().getNumNulls());
}
if (statistics.getBooleanStats().isSetNumTrues()) {
result.put("numTrues", statistics.getBooleanStats().getNumTrues());
}
if (statistics.getBooleanStats().isSetNumFalses()) {
result.put("numFalses", statistics.getBooleanStats().getNumFalses());
}
} else if (statistics.isSetDecimalStats()) {
if (statistics.getDecimalStats().isSetLowValue()) {
result.put("min", MetaDataFormatUtils.convertToString(statistics.getDecimalStats().getLowValue()));
}
if (statistics.getDecimalStats().isSetHighValue()) {
result.put("max", MetaDataFormatUtils.convertToString(statistics.getDecimalStats().getHighValue()));
}
if (statistics.getDecimalStats().isSetNumNulls()) {
result.put("numNulls", statistics.getDecimalStats().getNumNulls());
}
if (statistics.getDecimalStats().isSetNumDVs()) {
result.put("distinctCount", statistics.getDecimalStats().getNumDVs());
}
} else if (statistics.isSetDoubleStats()) {
if (statistics.getDoubleStats().isSetLowValue()) {
result.put("min", statistics.getDoubleStats().getLowValue());
}
if (statistics.getDoubleStats().isSetHighValue()) {
result.put("max", statistics.getDoubleStats().getHighValue());
}
if (statistics.getDoubleStats().isSetNumNulls()) {
result.put("numNulls", statistics.getDoubleStats().getNumNulls());
}
if (statistics.getDoubleStats().isSetNumDVs()) {
result.put("distinctCount", statistics.getDoubleStats().getNumDVs());
}
} else if (statistics.isSetLongStats()) {
if (statistics.getLongStats().isSetLowValue()) {
result.put("min", statistics.getLongStats().getLowValue());
}
if (statistics.getLongStats().isSetHighValue()) {
result.put("max", statistics.getLongStats().getHighValue());
}
if (statistics.getLongStats().isSetNumNulls()) {
result.put("numNulls", statistics.getLongStats().getNumNulls());
}
if (statistics.getLongStats().isSetNumDVs()) {
result.put("distinctCount", statistics.getLongStats().getNumDVs());
}
} else if (statistics.isSetDateStats()) {
if (statistics.getDateStats().isSetLowValue()) {
result.put("min", MetaDataFormatUtils.convertToString(statistics.getDateStats().getLowValue()));
}
if (statistics.getDateStats().isSetHighValue()) {
result.put("max", MetaDataFormatUtils.convertToString(statistics.getDateStats().getHighValue()));
}
if (statistics.getDateStats().isSetNumNulls()) {
result.put("numNulls", statistics.getDateStats().getNumNulls());
}
if (statistics.getDateStats().isSetNumDVs()) {
result.put("distinctCount", statistics.getDateStats().getNumDVs());
}
}
}

return result;
}

@Override
Expand Down Expand Up @@ -291,11 +396,11 @@ private Map<String, Object> makeOneTableStatus(Table tbl, Hive db,
builder.put("location", tblLoc);
builder.put("inputFormat", inputFormattCls);
builder.put("outputFormat", outputFormattCls);
builder.put("columns", makeColsUnformatted(tbl.getCols()));
builder.put("columns", createColumnsInfo(tbl.getCols(), new ArrayList<ColumnStatisticsObj>()));

builder.put("partitioned", tbl.isPartitioned());
if (tbl.isPartitioned()) {
builder.put("partitionColumns", makeColsUnformatted(tbl.getPartCols()));
builder.put("partitionColumns", createColumnsInfo(tbl.getPartCols(), new ArrayList<ColumnStatisticsObj>()));
}
if(tbl.getTableType() != TableType.VIRTUAL_VIEW) {
//tbl.getPath() is null for views
Expand Down
4 changes: 2 additions & 2 deletions ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java
View file
Open in desktop
Expand Up @@ -90,7 +90,7 @@ public final class MetaDataFormatUtils {
private MetaDataFormatUtils() {
}

private static String convertToString(Decimal val) {
public static String convertToString(Decimal val) {
if (val == null) {
return "";
}
Expand All @@ -103,7 +103,7 @@ private static String convertToString(Decimal val) {
}
}

private static String convertToString(org.apache.hadoop.hive.metastore.api.Date val) {
public static String convertToString(org.apache.hadoop.hive.metastore.api.Date val) {
if (val == null) {
return "";
}
Expand Down
27 changes: 15 additions & 12 deletions ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java
View file
Open in desktop
Expand Up @@ -33,7 +33,6 @@
import org.apache.hive.common.util.HiveStringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
Expand Down Expand Up @@ -139,7 +138,9 @@ public void showTablesExtended(DataOutputStream out, List<Table> tables)

try {
TextMetaDataTable mdt = new TextMetaDataTable();
mdt.addRow("# Table Name", "Table Type");
if (!SessionState.get().isHiveServerQuery()) {
mdt.addRow("# Table Name", "Table Type");
}
for (Table table : tables) {
final String tableName = table.getTableName();
final String tableType = table.getTableType().toString();
Expand Down Expand Up @@ -167,7 +168,9 @@ public void showMaterializedViews(DataOutputStream out, List<Table> materialized

try {
TextMetaDataTable mdt = new TextMetaDataTable();
mdt.addRow("# MV Name", "Rewriting Enabled", "Mode");
if (!SessionState.get().isHiveServerQuery()) {
mdt.addRow("# MV Name", "Rewriting Enabled", "Mode");
}
for (Table mv : materializedViews) {
final String mvName = mv.getTableName();
final String rewriteEnabled = mv.isRewriteEnabled() ? "Yes" : "No";
Expand Down Expand Up @@ -202,27 +205,27 @@ public void showMaterializedViews(DataOutputStream out, List<Table> materialized
}

@Override
public void describeTable(DataOutputStream outStream, String colPath, String tableName, Table tbl, Partition part,
public void describeTable(DataOutputStream outStream, String colPath, String tableName, Table tbl, Partition part,
List<FieldSchema> cols, boolean isFormatted, boolean isExt, boolean isOutputPadded,
List<ColumnStatisticsObj> colStats) throws HiveException {
try {
List<FieldSchema> partCols = tbl.isPartitioned() ? tbl.getPartCols() : null;
String output = "";

boolean isColStatsAvailable = CollectionUtils.isNotEmpty(colStats);
boolean needColStats = isFormatted && colPath != null;

TextMetaDataTable mdt = new TextMetaDataTable();
if (isFormatted && !isColStatsAvailable) {
if (needColStats) {
mdt.addRow(DescTableDesc.COLUMN_STATISTICS_HEADERS.toArray(new String[]{}));
} else if (isFormatted && !SessionState.get().isHiveServerQuery()) {
output = "# ";
}
if (isFormatted) {
mdt.addRow(DescTableDesc.getSchema(isColStatsAvailable).split("#")[0].split(","));
mdt.addRow(DescTableDesc.SCHEMA.split("#")[0].split(","));
}
for (FieldSchema col : cols) {
mdt.addRow(MetaDataFormatUtils.extractColumnValues(col, isColStatsAvailable,
mdt.addRow(MetaDataFormatUtils.extractColumnValues(col, needColStats,
MetaDataFormatUtils.getColumnStatisticsObject(col.getName(), col.getType(), colStats)));
}
if (isColStatsAvailable) {
if (needColStats) {
mdt.transpose();
}
output += mdt.renderTable(isOutputPadded);
Expand All @@ -231,7 +234,7 @@ public void describeTable(DataOutputStream outStream, String colPath, String ta
if ((partCols != null) && !partCols.isEmpty() && showPartColsSeparately) {
mdt = new TextMetaDataTable();
output += MetaDataFormatUtils.LINE_DELIM + "# Partition Information" + MetaDataFormatUtils.LINE_DELIM + "# ";
mdt.addRow(DescTableDesc.getSchema(false).split("#")[0].split(","));
mdt.addRow(DescTableDesc.SCHEMA.split("#")[0].split(","));
for (FieldSchema col : partCols) {
mdt.addRow(MetaDataFormatUtils.extractColumnValues(col));
}
Expand Down
7 changes: 4 additions & 3 deletions ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
View file
Open in desktop
Expand Up @@ -2258,10 +2258,11 @@ public QueryState getQueryState() {

/**
* Create a FetchTask for a given schema.
*
* @param schema string
*/
protected FetchTask createFetchTask(String schema) {
protected FetchTask createFetchTask(String tableSchema) {
String schema =
"json".equals(conf.get(HiveConf.ConfVars.HIVE_DDL_OUTPUT_FORMAT.varname, "text")) ? "json#string" : tableSchema;

Properties prop = new Properties();
// Sets delimiter to tab (ascii 9)
prop.setProperty(serdeConstants.SERIALIZATION_FORMAT, Integer.toString(Utilities.tabCode));
Expand Down
2 changes: 1 addition & 1 deletion ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
View file
Open in desktop
Expand Up @@ -1683,7 +1683,7 @@ private void analyzeDescribeTable(ASTNode ast) throws SemanticException {
DescTableDesc descTblDesc = new DescTableDesc(ctx.getResFile(), tableName, partSpec, colPath, isExt, isFormatted);
Task<?> ddlTask = TaskFactory.get(new DDLWork(getInputs(), getOutputs(), descTblDesc));
rootTasks.add(ddlTask);
String schema = DescTableDesc.getSchema(showColStats);
String schema = showColStats ? DescTableDesc.COLUMN_STATISTICS_SCHEMA : DescTableDesc.SCHEMA;
setFetchTask(createFetchTask(schema));
LOG.info("analyzeDescribeTable done");
}
Expand Down