From 7313aa0cf7b790ac4a3839808b716244523c932f Mon Sep 17 00:00:00 2001 From: Wenchen Fan Date: Fri, 11 Oct 2019 21:16:34 +0800 Subject: [PATCH] DDL commands should not use DataSourceV2Relation --- .../sql/catalyst/analysis/Analyzer.scala | 2 +- .../catalyst/analysis/ResolvedV2Table.scala | 34 +++++++++++++++++++ .../sql/catalyst/analysis/unresolved.scala | 2 +- .../datasources/v2/DataSourceV2Strategy.scala | 7 ++-- 4 files changed, 40 insertions(+), 5 deletions(-) create mode 100644 sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolvedV2Table.scala diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index b4d159eab4508..69dc07fc5a034 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -684,7 +684,7 @@ class Analyzer( case u: UnresolvedV2Relation => CatalogV2Util.loadTable(u.catalog, u.tableName).map { table => - DataSourceV2Relation.create(table) + ResolvedV2Table.create(table) }.getOrElse(u) } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolvedV2Table.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolvedV2Table.scala new file mode 100644 index 0000000000000..b6c4b848b0e86 --- /dev/null +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolvedV2Table.scala @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.analysis + +import org.apache.spark.sql.catalyst.expressions.AttributeReference +import org.apache.spark.sql.catalyst.plans.logical.LeafNode +import org.apache.spark.sql.connector.catalog.Table + +case class ResolvedV2Table(table: Table, output: Seq[AttributeReference]) + extends LeafNode with NamedRelation { + + override def name: String = table.name() +} + +object ResolvedV2Table { + def create(table: Table): ResolvedV2Table = { + ResolvedV2Table(table, table.schema().toAttributes) + } +} diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala index e5a6f30c330e6..af8c23b041cfc 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala @@ -62,7 +62,7 @@ object UnresolvedRelation { /** * A variant of [[UnresolvedRelation]] which can only be resolved to a v2 relation - * (`DataSourceV2Relation`), not v1 relation or temp view. + * (e.g. `ResolvedV2Table`), not v1 relation or temp view. * * @param originalNameParts the original table identifier name parts before catalog is resolved. * @param catalog The catalog which the table should be looked up from. diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala index c8d29520bcfce..743351860c6a8 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala @@ -21,6 +21,7 @@ import scala.collection.JavaConverters._ import scala.collection.mutable import org.apache.spark.sql.{AnalysisException, Strategy} +import org.apache.spark.sql.catalyst.analysis.ResolvedV2Table import org.apache.spark.sql.catalyst.expressions.{And, AttributeReference, AttributeSet, Expression, PredicateHelper, SubqueryExpression} import org.apache.spark.sql.catalyst.planning.PhysicalOperation import org.apache.spark.sql.catalyst.plans.logical.{AlterTable, AppendData, CreateTableAsSelect, CreateV2Table, DeleteFromTable, DescribeTable, DropTable, LogicalPlan, OverwriteByExpression, OverwritePartitionsDynamic, Repartition, ReplaceTable, ReplaceTableAsSelect, SetCatalogAndNamespace, ShowNamespaces, ShowTables} @@ -251,7 +252,7 @@ object DataSourceV2Strategy extends Strategy with PredicateHelper { OverwritePartitionsDynamicExec( r.table.asWritable, writeOptions.asOptions, planLater(query)) :: Nil - case DeleteFromTable(r: DataSourceV2Relation, condition) => + case DeleteFromTable(r: ResolvedV2Table, condition) => if (condition.exists(SubqueryExpression.hasSubquery)) { throw new AnalysisException( s"Delete by condition with subquery is not supported: $condition") @@ -280,8 +281,8 @@ object DataSourceV2Strategy extends Strategy with PredicateHelper { Nil } - case desc @ DescribeTable(r: DataSourceV2Relation, isExtended) => - DescribeTableExec(desc.output, r.table, isExtended) :: Nil + case desc @ DescribeTable(ResolvedV2Table(table, _), isExtended) => + DescribeTableExec(desc.output, table, isExtended) :: Nil case DropTable(catalog, ident, ifExists) => DropTableExec(catalog, ident, ifExists) :: Nil