Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
[SPARK-33634][SQL][TESTS] Use Analyzer in PlanResolutionSuite
### What changes were proposed in this pull request?

Instead of using several analyzer rules, this PR uses the actual analyzer to run tests in `PlanResolutionSuite`.

### Why are the changes needed?

Make the test suite to match reality.

### Does this PR introduce _any_ user-facing change?

no

### How was this patch tested?

test-only

Closes #30574 from cloud-fan/test.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
  • Loading branch information
cloud-fan authored and dongjoon-hyun committed Dec 3, 2020
1 parent aa13e20 commit 63f9d47
Showing 1 changed file with 14 additions and 21 deletions.
Expand Up @@ -26,14 +26,16 @@ import org.mockito.invocation.InvocationOnMock

import org.apache.spark.sql.{AnalysisException, SaveMode}
import org.apache.spark.sql.catalyst.{AliasIdentifier, TableIdentifier}
import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, Analyzer, CTESubstitution, EmptyFunctionRegistry, NoSuchTableException, ResolveCatalogs, ResolvedTable, ResolveInlineTables, ResolveSessionCatalog, UnresolvedAttribute, UnresolvedRelation, UnresolvedSubqueryColumnAliases, UnresolvedV2Relation}
import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, Analyzer, EmptyFunctionRegistry, NoSuchTableException, ResolvedTable, ResolveSessionCatalog, UnresolvedAttribute, UnresolvedRelation, UnresolvedSubqueryColumnAliases, UnresolvedV2Relation}
import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogStorageFormat, CatalogTable, CatalogTableType, InMemoryCatalog, SessionCatalog}
import org.apache.spark.sql.catalyst.expressions.{AttributeReference, EqualTo, Expression, InSubquery, IntegerLiteral, ListQuery, StringLiteral}
import org.apache.spark.sql.catalyst.parser.{CatalystSqlParser, ParseException}
import org.apache.spark.sql.catalyst.plans.logical.{AlterTable, Assignment, CreateTableAsSelect, CreateTableStatement, CreateV2Table, DeleteAction, DeleteFromTable, DescribeRelation, DropTable, InsertAction, InsertIntoStatement, LocalRelation, LogicalPlan, MergeIntoTable, OneRowRelation, Project, ShowTableProperties, SubqueryAlias, UpdateAction, UpdateTable}
import org.apache.spark.sql.catalyst.plans.logical.{AlterTable, AppendData, Assignment, CreateTableAsSelect, CreateTableStatement, CreateV2Table, DeleteAction, DeleteFromTable, DescribeRelation, DropTable, InsertAction, LocalRelation, LogicalPlan, MergeIntoTable, OneRowRelation, Project, ShowTableProperties, SubqueryAlias, UpdateAction, UpdateTable}
import org.apache.spark.sql.catalyst.rules.Rule
import org.apache.spark.sql.connector.FakeV2Provider
import org.apache.spark.sql.connector.catalog.{CatalogManager, CatalogNotFoundException, Identifier, Table, TableCapability, TableCatalog, TableChange, V1Table}
import org.apache.spark.sql.connector.catalog.TableChange.{UpdateColumnComment, UpdateColumnType}
import org.apache.spark.sql.connector.expressions.Transform
import org.apache.spark.sql.execution.datasources.CreateTable
import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
import org.apache.spark.sql.internal.{HiveSerDe, SQLConf}
Expand All @@ -49,6 +51,7 @@ class PlanResolutionSuite extends AnalysisTest {
private val table: Table = {
val t = mock(classOf[Table])
when(t.schema()).thenReturn(new StructType().add("i", "int").add("s", "string"))
when(t.partitioning()).thenReturn(Array.empty[Transform])
t
}

Expand Down Expand Up @@ -151,22 +154,12 @@ class PlanResolutionSuite extends AnalysisTest {
} else {
catalogManagerWithoutDefault
}
val analyzer = new Analyzer(catalogManager)
// TODO: run the analyzer directly.
val rules = Seq(
CTESubstitution,
ResolveInlineTables,
analyzer.ResolveRelations,
new ResolveCatalogs(catalogManager),
new ResolveSessionCatalog(catalogManager, _ == Seq("v"), _ => false),
analyzer.ResolveTables,
analyzer.ResolveReferences,
analyzer.ResolveSubqueryColumnAliases,
analyzer.ResolveReferences,
analyzer.ResolveAlterTableChanges)
rules.foldLeft(parsePlan(query)) {
case (plan, rule) => rule.apply(plan)
val analyzer = new Analyzer(catalogManager) {
override val extendedResolutionRules: Seq[Rule[LogicalPlan]] = Seq(
new ResolveSessionCatalog(catalogManager, _ == Seq("v"), _ => false))
}
// We don't check analysis here, as we expect the plan to be unresolved such as `CreateTable`.
analyzer.execute(CatalystSqlParser.parsePlan(query))
}

private def parseResolveCompare(query: String, expected: LogicalPlan): Unit =
Expand Down Expand Up @@ -1156,9 +1149,9 @@ class PlanResolutionSuite extends AnalysisTest {
("ALTER TABLE testcat.tab ALTER COLUMN i TYPE bigint", false),
("ALTER TABLE tab ALTER COLUMN i TYPE bigint", false),
(s"ALTER TABLE $v2SessionCatalogTable ALTER COLUMN i TYPE bigint", true),
("INSERT INTO TABLE tab VALUES (1)", false),
("INSERT INTO TABLE testcat.tab VALUES (1)", false),
(s"INSERT INTO TABLE $v2SessionCatalogTable VALUES (1)", true),
("INSERT INTO TABLE tab VALUES (1, 'a')", false),
("INSERT INTO TABLE testcat.tab VALUES (1, 'a')", false),
(s"INSERT INTO TABLE $v2SessionCatalogTable VALUES (1, 'a')", true),
("DESC TABLE tab", false),
("DESC TABLE testcat.tab", false),
(s"DESC TABLE $v2SessionCatalogTable", true),
Expand All @@ -1183,7 +1176,7 @@ class PlanResolutionSuite extends AnalysisTest {
case Project(_, AsDataSourceV2Relation(r)) =>
assert(r.catalog.exists(_ == catlogIdent))
assert(r.identifier.exists(_.name() == tableIdent))
case InsertIntoStatement(r: DataSourceV2Relation, _, _, _, _, _) =>
case AppendData(r: DataSourceV2Relation, _, _, _) =>
assert(r.catalog.exists(_ == catlogIdent))
assert(r.identifier.exists(_.name() == tableIdent))
case DescribeRelation(r: ResolvedTable, _, _) =>
Expand Down

0 comments on commit 63f9d47

Please sign in to comment.