Skip to content
Permalink
Browse files

[SPARK-29592][SQL] ALTER TABLE (set partition location) should look u…

…p catalog/table like v2 commands

### What changes were proposed in this pull request?

Update `AlterTableSetLocationStatement` to store `partitionSpec` and make `ALTER TABLE a.b.c PARTITION(...) SET LOCATION 'loc'` fail if `partitionSpec` is set with unsupported message.

### Why are the changes needed?

It's important to make all the commands have the same table resolution behavior, to avoid confusing end-users. e.g.

```
USE my_catalog
DESC t // success and describe the table t from my_catalog
ALTER TABLE t PARTITION(...) SET LOCATION 'loc' // report set location with partition spec is not supported.
```
### Does this PR introduce any user-facing change?

yes. When running ALTER TABLE (set partition location), Spark fails the command if the current catalog is set to a v2 catalog, or the table name specified a v2 catalog.

### How was this patch tested?

New unit tests

Closes #26304 from imback82/alter_table_partition_loc.

Authored-by: Terry Kim <yuminkim@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
  • Loading branch information
imback82 authored and cloud-fan committed Oct 31, 2019
1 parent 401a5f7 commit 3a06c129f4b6819c1b42c02d2c7c271376c7d22c
@@ -168,8 +168,8 @@ statement
DROP (IF EXISTS)? partitionSpec (',' partitionSpec)* PURGE? #dropTablePartitions
| ALTER VIEW tableIdentifier
DROP (IF EXISTS)? partitionSpec (',' partitionSpec)* #dropTablePartitions
| ALTER TABLE multipartIdentifier SET locationSpec #setTableLocation
| ALTER TABLE tableIdentifier partitionSpec SET locationSpec #setPartitionLocation
| ALTER TABLE multipartIdentifier
(partitionSpec)? SET locationSpec #setTableLocation
| ALTER TABLE multipartIdentifier RECOVER PARTITIONS #recoverPartitions
| DROP TABLE (IF EXISTS)? multipartIdentifier PURGE? #dropTable
| DROP VIEW (IF EXISTS)? multipartIdentifier #dropView
@@ -73,7 +73,11 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
createAlterTable(nameParts, catalog, tableName, changes)

case AlterTableSetLocationStatement(
nameParts @ NonSessionCatalog(catalog, tableName), newLoc) =>
nameParts @ NonSessionCatalog(catalog, tableName), partitionSpec, newLoc) =>
if (partitionSpec.nonEmpty) {
throw new AnalysisException(
"ALTER TABLE SET LOCATION does not support partition for v2 tables.")
}
val changes = Seq(TableChange.setProperty("location", newLoc))
createAlterTable(nameParts, catalog, tableName, changes)

@@ -2727,6 +2727,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
override def visitSetTableLocation(ctx: SetTableLocationContext): LogicalPlan = withOrigin(ctx) {
AlterTableSetLocationStatement(
visitMultipartIdentifier(ctx.multipartIdentifier),
Option(ctx.partitionSpec).map(visitNonOptionalPartitionSpec),
visitLocationSpec(ctx.locationSpec))
}

@@ -179,6 +179,7 @@ case class AlterTableUnsetPropertiesStatement(
*/
case class AlterTableSetLocationStatement(
tableName: Seq[String],
partitionSpec: Option[TablePartitionSpec],
location: String) extends ParsedStatement

/**
@@ -540,10 +540,16 @@ class DDLParserSuite extends AnalysisTest {
}

test("alter table: set location") {
val sql1 = "ALTER TABLE table_name SET LOCATION 'new location'"
val parsed1 = parsePlan(sql1)
val expected1 = AlterTableSetLocationStatement(Seq("table_name"), "new location")
comparePlans(parsed1, expected1)
comparePlans(
parsePlan("ALTER TABLE a.b.c SET LOCATION 'new location'"),
AlterTableSetLocationStatement(Seq("a", "b", "c"), None, "new location"))

comparePlans(
parsePlan("ALTER TABLE a.b.c PARTITION(ds='2017-06-10') SET LOCATION 'new location'"),
AlterTableSetLocationStatement(
Seq("a", "b", "c"),
Some(Map("ds" -> "2017-06-10")),
"new location"))
}

test("alter table: rename column") {
@@ -118,11 +118,15 @@ class ResolveSessionCatalog(
}

case AlterTableSetLocationStatement(
nameParts @ SessionCatalog(catalog, tableName), newLoc) =>
nameParts @ SessionCatalog(catalog, tableName), partitionSpec, newLoc) =>
loadTable(catalog, tableName.asIdentifier).collect {
case v1Table: V1Table =>
AlterTableSetLocationCommand(tableName.asTableIdentifier, None, newLoc)
AlterTableSetLocationCommand(tableName.asTableIdentifier, partitionSpec, newLoc)
}.getOrElse {
if (partitionSpec.nonEmpty) {
throw new AnalysisException(
"ALTER TABLE SET LOCATION does not support partition for v2 tables.")
}
val changes = Seq(TableChange.setProperty("location", newLoc))
createAlterTable(nameParts, catalog, tableName, changes)
}
@@ -502,22 +502,6 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) {
retainData = false)
}

/**
* Create an [[AlterTableSetLocationCommand]] command for a partition.
*
* For example:
* {{{
* ALTER TABLE table PARTITION spec SET LOCATION "loc";
* }}}
*/
override def visitSetPartitionLocation(
ctx: SetPartitionLocationContext): LogicalPlan = withOrigin(ctx) {
AlterTableSetLocationCommand(
visitTableIdentifier(ctx.tableIdentifier),
Some(visitNonOptionalPartitionSpec(ctx.partitionSpec)),
visitLocationSpec(ctx.locationSpec))
}

/**
* Create a [[AlterTableChangeColumnCommand]] command.
*
@@ -816,6 +816,19 @@ trait AlterTableTests extends SharedSparkSession {
}
}

test("AlterTable: set partition location") {
val t = s"${catalogAndNamespace}table_name"
withTable(t) {
sql(s"CREATE TABLE $t (id int) USING $v2Format")

val exc = intercept[AnalysisException] {
sql(s"ALTER TABLE $t PARTITION(ds='2017-06-10') SET LOCATION 's3://bucket/path'")
}
assert(exc.getMessage.contains(
"ALTER TABLE SET LOCATION does not support partition for v2 tables"))
}
}

test("AlterTable: set table property") {
val t = s"${catalogAndNamespace}table_name"
withTable(t) {
@@ -136,14 +136,21 @@ abstract class SQLViewSuite extends QueryTest with SQLTestUtils {
assertNoSuchTable(s"ALTER TABLE $viewName SET SERDE 'whatever'")
assertNoSuchTable(s"ALTER TABLE $viewName PARTITION (a=1, b=2) SET SERDE 'whatever'")
assertNoSuchTable(s"ALTER TABLE $viewName SET SERDEPROPERTIES ('p' = 'an')")
assertNoSuchTable(s"ALTER TABLE $viewName PARTITION (a='4') SET LOCATION '/path/to/home'")
assertNoSuchTable(s"ALTER TABLE $viewName ADD IF NOT EXISTS PARTITION (a='4', b='8')")
assertNoSuchTable(s"ALTER TABLE $viewName DROP PARTITION (a='4', b='8')")
assertNoSuchTable(s"ALTER TABLE $viewName PARTITION (a='4') RENAME TO PARTITION (a='5')")
assertNoSuchTable(s"ALTER TABLE $viewName RECOVER PARTITIONS")

// For v2 ALTER TABLE statements, we have better error message saying view is not supported.
assertViewNotSupported(s"ALTER TABLE $viewName SET LOCATION '/path/to/your/lovely/heart'")
assertAnalysisError(
s"ALTER TABLE $viewName SET LOCATION '/path/to/your/lovely/heart'",
s"'$viewName' is a view not a table")

// For the following v2 ALERT TABLE statements, unsupported operations are checked first
// before resolving the relations.
assertAnalysisError(
s"ALTER TABLE $viewName PARTITION (a='4') SET LOCATION '/path/to/home'",
"ALTER TABLE SET LOCATION does not support partition for v2 tables")
}
}

@@ -177,9 +184,9 @@ abstract class SQLViewSuite extends QueryTest with SQLTestUtils {
}
}

private def assertViewNotSupported(query: String): Unit = {
private def assertAnalysisError(query: String, message: String): Unit = {
val e = intercept[AnalysisException](sql(query))
assert(e.message.contains("'testView' is a view not a table"))
assert(e.message.contains(message))
}

test("error handling: insert/load/truncate table commands against a view") {
@@ -635,18 +635,6 @@ class DDLParserSuite extends AnalysisTest with SharedSparkSession {
"SET FILEFORMAT PARQUET")
}

test("alter table: set partition location") {
val sql2 = "ALTER TABLE table_name PARTITION (dt='2008-08-08', country='us') " +
"SET LOCATION 'new location'"
val parsed2 = parser.parsePlan(sql2)
val tableIdent = TableIdentifier("table_name", None)
val expected2 = AlterTableSetLocationCommand(
tableIdent,
Some(Map("dt" -> "2008-08-08", "country" -> "us")),
"new location")
comparePlans(parsed2, expected2)
}

test("alter table: change column name/type/comment") {
val sql1 = "ALTER TABLE table_name CHANGE COLUMN col_old_name col_new_name INT"
val sql2 = "ALTER TABLE table_name CHANGE COLUMN col_name col_name INT COMMENT 'new_comment'"

0 comments on commit 3a06c12

Please sign in to comment.
You can’t perform that action at this time.