IMPALA-4403: Implement SHOW RANGE PARTITIONS for Kudu tables

Change-Id: Idf5b2fdd02938a42fa59ec98884e4ac915dd1f65 Reviewed-on: http://gerrit.cloudera.org:8080/5390 Reviewed-by: Lars Volker <lv@cloudera.com> Reviewed-by: Matthew Jacobs <mj@cloudera.com> Tested-by: Internal Jenkins
cloudera · Dec 10, 2016 · 45cb3b3 · 45cb3b3
1 parent 20d17bf
commit 45cb3b3
Show file tree

Hide file tree

Showing 12 changed files with 238 additions and 72 deletions.
diff --git a/common/thrift/Frontend.thrift b/common/thrift/Frontend.thrift
@@ -177,9 +177,17 @@ struct TShowDbsParams {
   1: optional string show_pattern
 }
 
-// Parameters for SHOW TABLE/COLUMN STATS commands
+// Used by SHOW STATS and SHOW PARTITIONS to control what information is returned.
+enum TShowStatsOp {
+  TABLE_STATS,
+  COLUMN_STATS,
+  PARTITIONS,
+  RANGE_PARTITIONS
+}
+
+// Parameters for SHOW TABLE/COLUMN STATS and SHOW PARTITIONS commands
 struct TShowStatsParams {
-  1: required bool is_show_col_stats
+  1: TShowStatsOp op
   2: CatalogObjects.TTableName table_name
 }
 

diff --git a/fe/src/main/cup/sql-parser.cup b/fe/src/main/cup/sql-parser.cup
@@ -49,6 +49,7 @@ import org.apache.impala.thrift.TDescribeOutputStyle;
 import org.apache.impala.thrift.TFunctionCategory;
 import org.apache.impala.thrift.THdfsFileFormat;
 import org.apache.impala.thrift.TPrivilegeLevel;
+import org.apache.impala.thrift.TShowStatsOp;
 import org.apache.impala.thrift.TTablePropertyType;
 
 parser code {:
@@ -308,8 +309,8 @@ nonterminal UseStmt use_stmt;
 nonterminal SetStmt set_stmt;
 nonterminal ShowTablesStmt show_tables_stmt;
 nonterminal ShowDbsStmt show_dbs_stmt;
-nonterminal ShowPartitionsStmt show_partitions_stmt;
-nonterminal ShowStatsStmt show_stats_stmt;
+nonterminal ShowStatsStmt show_stats_stmt, show_partitions_stmt,
+  show_range_partitions_stmt;
 nonterminal String show_pattern;
 nonterminal ShowFilesStmt show_files_stmt;
 nonterminal DescribeDbStmt describe_db_stmt;
@@ -552,6 +553,8 @@ stmt ::=
   {: RESULT = show_dbs; :}
   | show_partitions_stmt:show_partitions
   {: RESULT = show_partitions; :}
+  | show_range_partitions_stmt:show_range_partitions
+  {: RESULT = show_range_partitions; :}
   | show_stats_stmt:show_stats
   {: RESULT = show_stats; :}
   | show_functions_stmt:show_functions
@@ -2084,14 +2087,19 @@ show_dbs_stmt ::=
 
 show_stats_stmt ::=
   KW_SHOW KW_TABLE KW_STATS table_name:table
-  {: RESULT = new ShowStatsStmt(table, false); :}
+  {: RESULT = new ShowStatsStmt(table, TShowStatsOp.TABLE_STATS); :}
   | KW_SHOW KW_COLUMN KW_STATS table_name:table
-  {: RESULT = new ShowStatsStmt(table, true); :}
+  {: RESULT = new ShowStatsStmt(table, TShowStatsOp.COLUMN_STATS); :}
   ;
 
 show_partitions_stmt ::=
   KW_SHOW KW_PARTITIONS table_name:table
-  {: RESULT = new ShowPartitionsStmt(table); :}
+  {: RESULT = new ShowStatsStmt(table, TShowStatsOp.PARTITIONS); :}
+  ;
+
+show_range_partitions_stmt ::=
+  KW_SHOW KW_RANGE KW_PARTITIONS table_name:table
+  {: RESULT = new ShowStatsStmt(table, TShowStatsOp.RANGE_PARTITIONS); :}
   ;
 
 show_functions_stmt ::=

diff --git a/fe/src/main/java/org/apache/impala/analysis/ShowPartitionsStmt.java b/fe/src/main/java/org/apache/impala/analysis/ShowPartitionsStmt.java
diff --git a/fe/src/main/java/org/apache/impala/analysis/ShowStatsStmt.java b/fe/src/main/java/org/apache/impala/analysis/ShowStatsStmt.java
@@ -18,25 +18,29 @@
 package org.apache.impala.analysis;
 
 import org.apache.impala.authorization.Privilege;
+import org.apache.impala.catalog.HdfsTable;
+import org.apache.impala.catalog.KuduTable;
 import org.apache.impala.catalog.Table;
 import org.apache.impala.catalog.View;
 import org.apache.impala.common.AnalysisException;
+import org.apache.impala.thrift.TShowStatsOp;
 import org.apache.impala.thrift.TShowStatsParams;
+import com.google.common.base.Preconditions;
 
 /**
  * Representation of a SHOW TABLE/COLUMN STATS statement for
  * displaying column and table/partition statistics for a given table.
  */
 public class ShowStatsStmt extends StatementBase {
-  protected final boolean isShowColStats_;
+  protected final TShowStatsOp op_;
   protected final TableName tableName_;
 
   // Set during analysis.
   protected Table table_;
 
-  public ShowStatsStmt(TableName tableName, boolean isShowColStats) {
+  public ShowStatsStmt(TableName tableName, TShowStatsOp op) {
+    this.op_ = op;
     this.tableName_ = tableName;
-    this.isShowColStats_ = isShowColStats;
   }
 
   @Override
@@ -45,21 +49,57 @@ public String toSql() {
   }
 
   protected String getSqlPrefix() {
-    return "SHOW " + ((isShowColStats_) ? "COLUMN" : "TABLE") + " STATS";
+    if (op_ == TShowStatsOp.TABLE_STATS) {
+      return "SHOW TABLE STATS";
+    } else if (op_ == TShowStatsOp.COLUMN_STATS) {
+      return "SHOW COLUMN STATS";
+    } else if (op_ == TShowStatsOp.PARTITIONS) {
+      return "SHOW PARTITIONS";
+    } else if (op_ == TShowStatsOp.RANGE_PARTITIONS) {
+      return "SHOW RANGE PARTITIONS";
+    } else {
+      Preconditions.checkState(false);
+      return "";
+    }
   }
 
   @Override
   public void analyze(Analyzer analyzer) throws AnalysisException {
     table_ = analyzer.getTable(tableName_, Privilege.VIEW_METADATA);
+    Preconditions.checkNotNull(table_);
     if (table_ instanceof View) {
       throw new AnalysisException(String.format(
           "%s not applicable to a view: %s", getSqlPrefix(), table_.getFullName()));
     }
+    if (table_ instanceof HdfsTable) {
+      if (table_.getNumClusteringCols() == 0 && op_ == TShowStatsOp.PARTITIONS) {
+        throw new AnalysisException("Table is not partitioned: " + table_.getFullName());
+      }
+      if (op_ == TShowStatsOp.RANGE_PARTITIONS) {
+        throw new AnalysisException(getSqlPrefix() + " must target a Kudu table: " +
+            table_.getFullName());
+      }
+    } else if (table_ instanceof KuduTable) {
+      KuduTable kuduTable = (KuduTable) table_;
+      if (op_ == TShowStatsOp.RANGE_PARTITIONS &&
+          kuduTable.getRangePartitioningColNames().isEmpty()) {
+        throw new AnalysisException(getSqlPrefix() + " requested but table does not " +
+            "have range partitions: " + table_.getFullName());
+      }
+    } else {
+      if (op_ == TShowStatsOp.RANGE_PARTITIONS) {
+        throw new AnalysisException(getSqlPrefix() + " must target a Kudu table: " +
+            table_.getFullName());
+      } else if (op_ == TShowStatsOp.PARTITIONS) {
+        throw new AnalysisException(getSqlPrefix() + " must target an HDFS table: " +
+            table_.getFullName());
+      }
+    }
   }
 
   public TShowStatsParams toThrift() {
     // Ensure the DB is set in the table_name field by using table and not tableName.
-    return new TShowStatsParams(isShowColStats_,
+    return new TShowStatsParams(op_,
         new TableName(table_.getDb().getName(), table_.getName()).toThrift());
   }
 }
diff --git a/fe/src/main/java/org/apache/impala/catalog/KuduTable.java b/fe/src/main/java/org/apache/impala/catalog/KuduTable.java
@@ -413,4 +413,33 @@ public TResultSet getTableStats() throws ImpalaRuntimeException {
     }
     return result;
   }
+
+  public TResultSet getRangePartitions() throws ImpalaRuntimeException {
+    TResultSet result = new TResultSet();
+    TResultSetMetadata resultSchema = new TResultSetMetadata();
+    result.setSchema(resultSchema);
+
+    // Build column header
+    String header = "RANGE (" + Joiner.on(',').join(getRangePartitioningColNames()) + ")";
+    resultSchema.addToColumns(new TColumn(header, Type.STRING.toThrift()));
+    try (KuduClient client = KuduUtil.createKuduClient(getKuduMasterHosts())) {
+      org.apache.kudu.client.KuduTable kuduTable = client.openTable(kuduTableName_);
+      // The Kudu table API will return the partitions in sorted order by value.
+      List<String> partitions = kuduTable.getFormattedRangePartitions(
+          BackendConfig.INSTANCE.getKuduClientTimeoutMs());
+      if (partitions.isEmpty()) {
+        TResultRowBuilder builder = new TResultRowBuilder();
+        result.addToRows(builder.add("").get());
+        return result;
+      }
+      for (String partition: partitions) {
+        TResultRowBuilder builder = new TResultRowBuilder();
+        builder.add(partition);
+        result.addToRows(builder.get());
+      }
+    } catch (Exception e) {
+      throw new ImpalaRuntimeException("Error accessing Kudu for table partitions.", e);
+    }
+    return result;
+  }
 }
diff --git a/fe/src/main/java/org/apache/impala/service/Frontend.java b/fe/src/main/java/org/apache/impala/service/Frontend.java
@@ -120,6 +120,7 @@
 import org.apache.impala.thrift.TResultSet;
 import org.apache.impala.thrift.TResultSetMetadata;
 import org.apache.impala.thrift.TShowFilesParams;
+import org.apache.impala.thrift.TShowStatsOp;
 import org.apache.impala.thrift.TStatus;
 import org.apache.impala.thrift.TStmtType;
 import org.apache.impala.thrift.TTableName;
@@ -715,7 +716,7 @@ public TResultSet getColumnStats(String dbName, String tableName)
   /**
    * Generate result set and schema for a SHOW TABLE STATS command.
    */
-  public TResultSet getTableStats(String dbName, String tableName)
+  public TResultSet getTableStats(String dbName, String tableName, TShowStatsOp op)
       throws ImpalaException {
     Table table = impaladCatalog_.getTable(dbName, tableName);
     if (table instanceof HdfsTable) {
@@ -725,7 +726,11 @@ public TResultSet getTableStats(String dbName, String tableName)
     } else if (table instanceof DataSourceTable) {
       return ((DataSourceTable) table).getTableStats();
     } else if (table instanceof KuduTable) {
-      return ((KuduTable) table).getTableStats();
+      if (op == TShowStatsOp.RANGE_PARTITIONS) {
+        return ((KuduTable) table).getRangePartitions();
+      } else {
+        return ((KuduTable) table).getTableStats();
+      }
     } else {
       throw new InternalException("Invalid table class: " + table.getClass());
     }

diff --git a/fe/src/main/java/org/apache/impala/service/JniFrontend.java b/fe/src/main/java/org/apache/impala/service/JniFrontend.java
@@ -80,6 +80,7 @@
 import org.apache.impala.thrift.TShowGrantRoleParams;
 import org.apache.impala.thrift.TShowRolesParams;
 import org.apache.impala.thrift.TShowRolesResult;
+import org.apache.impala.thrift.TShowStatsOp;
 import org.apache.impala.thrift.TShowStatsParams;
 import org.apache.impala.thrift.TTableName;
 import org.apache.impala.thrift.TUniqueId;
@@ -358,12 +359,13 @@ public byte[] getStats(byte[] thriftShowStatsParams) throws ImpalaException {
     JniUtil.deserializeThrift(protocolFactory_, params, thriftShowStatsParams);
     Preconditions.checkState(params.isSetTable_name());
     TResultSet result;
-    if (params.isIs_show_col_stats()) {
+
+    if (params.op == TShowStatsOp.COLUMN_STATS) {
       result = frontend_.getColumnStats(params.getTable_name().getDb_name(),
           params.getTable_name().getTable_name());
     } else {
       result = frontend_.getTableStats(params.getTable_name().getDb_name(),
-          params.getTable_name().getTable_name());
+          params.getTable_name().getTable_name(), params.op);
     }
     TSerializer serializer = new TSerializer(protocolFactory_);
     try {

diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
@@ -3287,6 +3287,26 @@ public void TestShowPartitions() throws AnalysisException {
         "SHOW PARTITIONS must target an HDFS table: functional_hbase.alltypes");
   }
 
+  @Test
+  public void TestShowRangePartitions() throws AnalysisException {
+    AnalyzesOk("show range partitions functional_kudu.dimtbl");
+    AnalysisError("show range partitions baddb.alltypes",
+        "Database does not exist: baddb");
+    AnalysisError("show range partitions functional.badtbl",
+        "Table does not exist: functional.badtbl");
+    AnalysisError("show range partitions functional.alltypes",
+        "SHOW RANGE PARTITIONS must target a Kudu table: functional.alltypes");
+    AnalysisError("show range partitions functional.alltypesnopart",
+        "SHOW RANGE PARTITIONS must target a Kudu table: functional.alltypes");
+    AnalysisError("show range partitions functional_kudu.alltypes",
+        "SHOW RANGE PARTITIONS requested but table does not have range partitions: " +
+        "functional_kudu.alltypes");
+    AnalysisError("show range partitions functional.view_view",
+        "SHOW RANGE PARTITIONS not applicable to a view: functional.view_view");
+    AnalysisError("show range partitions functional_hbase.alltypes",
+        "SHOW RANGE PARTITIONS must target a Kudu table: functional_hbase.alltypes");
+  }
+
   @Test
   public void TestShowCreateFunction() throws AnalysisException {
     addTestFunction("TestFn", Lists.newArrayList(Type.INT, Type.INT), false);

diff --git a/fe/src/test/java/org/apache/impala/analysis/ParserTest.java b/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
@@ -1744,6 +1744,11 @@ public void TestShow() {
     ParsesOk("SHOW PARTITIONS db.tbl");
     ParsesOk("SHOW PARTITIONS `db`.`tbl`");
 
+    // Show range partitions
+    ParsesOk("SHOW RANGE PARTITIONS tbl");
+    ParsesOk("SHOW RANGE PARTITIONS db.tbl");
+    ParsesOk("SHOW RANGE PARTITIONS `db`.`tbl`");
+
     // Show files of table
     ParsesOk("SHOW FILES IN tbl");
     ParsesOk("SHOW FILES IN db.tbl");