Skip to content

Commit

Permalink
[SPARK-16926][SQL] Add unit test to compare table and partition colum…
Browse files Browse the repository at this point in the history
…n metadata.

## What changes were proposed in this pull request?

Add unit test for changes made in PR #14515. It makes sure that a newly created table has the same number of columns in table and partition metadata. This test fails before the changes introduced in #14515.

## How was this patch tested?

Run new unit test.

Author: Brian Cho <bcho@fb.com>

Closes #14930 from dafrista/partition-metadata-unit-test.
  • Loading branch information
bchocho authored and cloud-fan committed Sep 2, 2016
1 parent 06e3398 commit f2d6e2e
Showing 1 changed file with 35 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package org.apache.spark.sql.hive.execution

import org.apache.spark.sql.Row
import org.apache.spark.sql.hive.MetastoreRelation
import org.apache.spark.sql.hive.test.{TestHive, TestHiveSingleton}
import org.apache.spark.sql.hive.test.TestHive._
import org.apache.spark.sql.hive.test.TestHive.implicits._
Expand Down Expand Up @@ -143,4 +144,38 @@ class HiveTableScanSuite extends HiveComparisonTest with SQLTestUtils with TestH
}
}
}

test("SPARK-16926: number of table and partition columns match for new partitioned table") {
val view = "src"
withTempView(view) {
spark.range(1, 5).createOrReplaceTempView(view)
val table = "table_with_partition"
withTable(table) {
sql(
s"""
|CREATE TABLE $table(id string)
|PARTITIONED BY (p1 string,p2 string,p3 string,p4 string,p5 string)
""".stripMargin)
sql(
s"""
|FROM $view v
|INSERT INTO TABLE $table
|PARTITION (p1='a',p2='b',p3='c',p4='d',p5='e')
|SELECT v.id
|INSERT INTO TABLE $table
|PARTITION (p1='a',p2='c',p3='c',p4='d',p5='e')
|SELECT v.id
""".stripMargin)
val plan = sql(
s"""
|SELECT * FROM $table
""".stripMargin).queryExecution.sparkPlan
val relation = plan.collectFirst {
case p: HiveTableScanExec => p.relation
}.get
val tableCols = relation.hiveQlTable.getCols
relation.getHiveQlPartitions().foreach(p => assert(p.getCols.size == tableCols.size))
}
}
}
}

0 comments on commit f2d6e2e

Please sign in to comment.