-
Notifications
You must be signed in to change notification settings - Fork 28k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[SPARK-27845][SQL] DataSourceV2: InsertTable #24832
Changes from all commits
8a3b61d
efc4bf7
8449d66
97dc04c
7f193ca
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -294,8 +294,8 @@ query | |
; | ||
|
||
insertInto | ||
: INSERT OVERWRITE TABLE tableIdentifier (partitionSpec (IF NOT EXISTS)?)? #insertOverwriteTable | ||
| INSERT INTO TABLE? tableIdentifier partitionSpec? #insertIntoTable | ||
: INSERT OVERWRITE TABLE? multipartIdentifier (partitionSpec (IF NOT EXISTS)?)? #insertOverwriteTable | ||
| INSERT INTO TABLE? multipartIdentifier partitionSpec? (IF NOT EXISTS)? #insertIntoTable | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Actually, There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can we keep this unchanged and not add There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This was changed to get a better error message. Instead of a parse exception that lists symbols, this is now a useful error message with a test. |
||
| INSERT OVERWRITE LOCAL? DIRECTORY path=STRING rowFormat? createFileFormat? #insertOverwriteHiveDir | ||
| INSERT OVERWRITE LOCAL? DIRECTORY (path=STRING)? tableProvider (OPTIONS options=tablePropertyList)? #insertOverwriteDir | ||
; | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -38,7 +38,7 @@ import org.apache.spark.sql.catalyst.expressions.aggregate.{First, Last} | |
import org.apache.spark.sql.catalyst.parser.SqlBaseParser._ | ||
import org.apache.spark.sql.catalyst.plans._ | ||
import org.apache.spark.sql.catalyst.plans.logical._ | ||
import org.apache.spark.sql.catalyst.plans.logical.sql.{AlterTableAddColumnsStatement, AlterTableAlterColumnStatement, AlterTableDropColumnsStatement, AlterTableRenameColumnStatement, AlterTableSetLocationStatement, AlterTableSetPropertiesStatement, AlterTableUnsetPropertiesStatement, AlterViewSetPropertiesStatement, AlterViewUnsetPropertiesStatement, CreateTableAsSelectStatement, CreateTableStatement, DropTableStatement, DropViewStatement, QualifiedColType, ReplaceTableAsSelectStatement, ReplaceTableStatement} | ||
import org.apache.spark.sql.catalyst.plans.logical.sql.{AlterTableAddColumnsStatement, AlterTableAlterColumnStatement, AlterTableDropColumnsStatement, AlterTableRenameColumnStatement, AlterTableSetLocationStatement, AlterTableSetPropertiesStatement, AlterTableUnsetPropertiesStatement, AlterViewSetPropertiesStatement, AlterViewUnsetPropertiesStatement, CreateTableAsSelectStatement, CreateTableStatement, DropTableStatement, DropViewStatement, InsertIntoStatement, QualifiedColType, ReplaceTableAsSelectStatement, ReplaceTableStatement} | ||
import org.apache.spark.sql.catalyst.util.DateTimeUtils.{getZoneId, stringToDate, stringToTimestamp} | ||
import org.apache.spark.sql.internal.SQLConf | ||
import org.apache.spark.sql.types._ | ||
|
@@ -239,9 +239,9 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging | |
|
||
/** | ||
* Parameters used for writing query to a table: | ||
* (tableIdentifier, partitionKeys, exists). | ||
* (multipartIdentifier, partitionKeys, ifPartitionNotExists). | ||
*/ | ||
type InsertTableParams = (TableIdentifier, Map[String, Option[String]], Boolean) | ||
type InsertTableParams = (Seq[String], Map[String, Option[String]], Boolean) | ||
|
||
/** | ||
* Parameters used for writing query to a directory: (isLocal, CatalogStorageFormat, provider). | ||
|
@@ -263,11 +263,21 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging | |
query: LogicalPlan): LogicalPlan = withOrigin(ctx) { | ||
ctx match { | ||
case table: InsertIntoTableContext => | ||
val (tableIdent, partitionKeys, exists) = visitInsertIntoTable(table) | ||
InsertIntoTable(UnresolvedRelation(tableIdent), partitionKeys, query, false, exists) | ||
val (tableIdent, partition, ifPartitionNotExists) = visitInsertIntoTable(table) | ||
InsertIntoStatement( | ||
UnresolvedRelation(tableIdent), | ||
partition, | ||
query, | ||
overwrite = false, | ||
ifPartitionNotExists) | ||
case table: InsertOverwriteTableContext => | ||
val (tableIdent, partitionKeys, exists) = visitInsertOverwriteTable(table) | ||
InsertIntoTable(UnresolvedRelation(tableIdent), partitionKeys, query, true, exists) | ||
val (tableIdent, partition, ifPartitionNotExists) = visitInsertOverwriteTable(table) | ||
InsertIntoStatement( | ||
UnresolvedRelation(tableIdent), | ||
partition, | ||
query, | ||
overwrite = true, | ||
ifPartitionNotExists) | ||
case dir: InsertOverwriteDirContext => | ||
val (isLocal, storage, provider) = visitInsertOverwriteDir(dir) | ||
InsertIntoDir(isLocal, storage, provider, query, overwrite = true) | ||
|
@@ -284,9 +294,13 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging | |
*/ | ||
override def visitInsertIntoTable( | ||
ctx: InsertIntoTableContext): InsertTableParams = withOrigin(ctx) { | ||
val tableIdent = visitTableIdentifier(ctx.tableIdentifier) | ||
val tableIdent = visitMultipartIdentifier(ctx.multipartIdentifier) | ||
val partitionKeys = Option(ctx.partitionSpec).map(visitPartitionSpec).getOrElse(Map.empty) | ||
|
||
if (ctx.EXISTS != null) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. what's the point of adding this to the parser, if we're not going to support it? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. For a better error message that is testable. Before, there were no tests for this case and the error message listed expected symbols. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Also, since the PARTITION clause is optional for the above case, it shouldn't group the two together either. It is semantically incorrect because a write to a partitioned table is always a partitioned write. |
||
operationNotAllowed("INSERT INTO ... IF NOT EXISTS", ctx) | ||
} | ||
|
||
(tableIdent, partitionKeys, false) | ||
} | ||
|
||
|
@@ -296,13 +310,13 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging | |
override def visitInsertOverwriteTable( | ||
ctx: InsertOverwriteTableContext): InsertTableParams = withOrigin(ctx) { | ||
assert(ctx.OVERWRITE() != null) | ||
val tableIdent = visitTableIdentifier(ctx.tableIdentifier) | ||
val tableIdent = visitMultipartIdentifier(ctx.multipartIdentifier) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think this needs to be updated to remove the Also, I don't see a reason why IF NOT EXISTS would not be supported with dynamic partitions. Wouldn't that fail if any partitions would be overwritten? It seems to make sense to me, but maybe there is a good reason why this is not allowed? @gatorsmile can you comment? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We discussed this in the DSv2 sync last night and decided to add a method to the write builder to pass this There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
val partitionKeys = Option(ctx.partitionSpec).map(visitPartitionSpec).getOrElse(Map.empty) | ||
|
||
val dynamicPartitionKeys: Map[String, Option[String]] = partitionKeys.filter(_._2.isEmpty) | ||
if (ctx.EXISTS != null && dynamicPartitionKeys.nonEmpty) { | ||
throw new ParseException(s"Dynamic partitions do not support IF NOT EXISTS. Specified " + | ||
"partitions with value: " + dynamicPartitionKeys.keys.mkString("[", ",", "]"), ctx) | ||
operationNotAllowed("IF NOT EXISTS with dynamic partitions: " + | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why do we change the error message here? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This uses |
||
dynamicPartitionKeys.keys.mkString(", "), ctx) | ||
} | ||
|
||
(tableIdent, partitionKeys, ctx.EXISTS() != null) | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,50 @@ | ||
/* | ||
* Licensed to the Apache Software Foundation (ASF) under one or more | ||
* contributor license agreements. See the NOTICE file distributed with | ||
* this work for additional information regarding copyright ownership. | ||
* The ASF licenses this file to You under the Apache License, Version 2.0 | ||
* (the "License"); you may not use this file except in compliance with | ||
* the License. You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
|
||
package org.apache.spark.sql.catalyst.plans.logical.sql | ||
|
||
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan | ||
|
||
/** | ||
* An INSERT INTO statement, as parsed from SQL. | ||
* | ||
* @param table the logical plan representing the table. | ||
* @param query the logical plan representing data to write to. | ||
* @param overwrite overwrite existing table or partitions. | ||
* @param partitionSpec a map from the partition key to the partition value (optional). | ||
* If the value is missing, dynamic partition insert will be performed. | ||
* As an example, `INSERT INTO tbl PARTITION (a=1, b=2) AS` would have | ||
* Map('a' -> Some('1'), 'b' -> Some('2')), | ||
* and `INSERT INTO tbl PARTITION (a=1, b) AS ...` | ||
* would have Map('a' -> Some('1'), 'b' -> None). | ||
* @param ifPartitionNotExists If true, only write if the partition does not exist. | ||
* Only valid for static partitions. | ||
*/ | ||
case class InsertIntoStatement( | ||
table: LogicalPlan, | ||
partitionSpec: Map[String, Option[String]], | ||
query: LogicalPlan, | ||
overwrite: Boolean, | ||
ifPartitionNotExists: Boolean) extends ParsedStatement { | ||
|
||
require(overwrite || !ifPartitionNotExists, | ||
"IF NOT EXISTS is only valid in INSERT OVERWRITE") | ||
require(partitionSpec.values.forall(_.nonEmpty) || !ifPartitionNotExists, | ||
"IF NOT EXISTS is only valid with static partitions") | ||
|
||
override def children: Seq[LogicalPlan] = query :: Nil | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
do we need to wrap with parentheses
(partitionSpec (IF NOT EXISTS)?)?
like above? Otherwise, what happens if there's nopartitionSpec
but theIF NOT EXISTS
?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It isn't supported either way, so why combine the two?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
got it