Skip to content
Permalink
Browse files

[SPARK-29922][SQL] SHOW FUNCTIONS should do multi-catalog resolution

### What changes were proposed in this pull request?

Add ShowFunctionsStatement and make SHOW FUNCTIONS go through the same catalog/table resolution framework of v2 commands.

We don’t have this methods in the catalog to implement an V2 command
* catalog.listFunctions

### Why are the changes needed?

It's important to make all the commands have the same table resolution behavior, to avoid confusing
`SHOW FUNCTIONS LIKE namespace.function`

### Does this PR introduce any user-facing change?

Yes. When running SHOW FUNCTIONS LIKE namespace.function Spark fails the command if the current catalog is set to a v2 catalog.

### How was this patch tested?

Unit tests.

Closes #26667 from planga82/feature/SPARK-29922_ShowFunctions_V2Catalog.

Authored-by: Pablo Langa <soypab@gmail.com>
Signed-off-by: Liang-Chi Hsieh <liangchi@uber.com>
  • Loading branch information
planga82 authored and viirya committed Dec 9, 2019
1 parent 16f1b23 commit bca9de66847dab562d44d65a284bf75e7ede6421
@@ -3358,4 +3358,20 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
visitMultipartIdentifier(ctx.table),
Option(ctx.key).map(visitTablePropertyKey))
}

/**
* Create a plan for a SHOW FUNCTIONS command.
*/
override def visitShowFunctions(ctx: ShowFunctionsContext): LogicalPlan = withOrigin(ctx) {
val (userScope, systemScope) = Option(ctx.identifier)
.map(_.getText.toLowerCase(Locale.ROOT)) match {
case None | Some("all") => (true, true)
case Some("system") => (false, true)
case Some("user") => (true, false)
case Some(x) => throw new ParseException(s"SHOW $x FUNCTIONS not supported", ctx)
}
val pattern = Option(ctx.pattern).map(string(_))
val functionName = Option(ctx.multipartIdentifier).map(visitMultipartIdentifier)
ShowFunctionsStatement(userScope, systemScope, pattern, functionName)
}
}
@@ -485,3 +485,12 @@ case class ShowCurrentNamespaceStatement() extends ParsedStatement
case class ShowTablePropertiesStatement(
tableName: Seq[String],
propertyKey: Option[String]) extends ParsedStatement

/**
* SHOW FUNCTIONS statement, as parsed from SQL
*/
case class ShowFunctionsStatement(
userScope: Boolean,
systemScope: Boolean,
pattern: Option[String],
functionName: Option[Seq[String]]) extends ParsedStatement
@@ -1799,6 +1799,32 @@ class DDLParserSuite extends AnalysisTest {
ShowTablePropertiesStatement(Seq("a", "b", "c"), Some("propKey1")))
}

test("SHOW FUNCTIONS") {
comparePlans(
parsePlan("SHOW FUNCTIONS"),
ShowFunctionsStatement(true, true, None, None))
comparePlans(
parsePlan("SHOW USER FUNCTIONS"),
ShowFunctionsStatement(true, false, None, None))
comparePlans(
parsePlan("SHOW user FUNCTIONS"),
ShowFunctionsStatement(true, false, None, None))
comparePlans(
parsePlan("SHOW SYSTEM FUNCTIONS"),
ShowFunctionsStatement(false, true, None, None))
comparePlans(
parsePlan("SHOW ALL FUNCTIONS"),
ShowFunctionsStatement(true, true, None, None))
comparePlans(
parsePlan("SHOW FUNCTIONS LIKE 'funct*'"),
ShowFunctionsStatement(true, true, Some("funct*"), None))
comparePlans(
parsePlan("SHOW FUNCTIONS LIKE a.b.c"),
ShowFunctionsStatement(true, true, None, Some(Seq("a", "b", "c"))))
val sql = "SHOW other FUNCTIONS"
intercept(sql, s"$sql not supported")
}

private case class TableSpec(
name: Seq[String],
schema: Option[StructType],
@@ -471,6 +471,23 @@ class ResolveSessionCatalog(
ShowTablePropertiesCommand(
tableName.asTableIdentifier,
propertyKey)

case ShowFunctionsStatement(userScope, systemScope, pattern, fun) =>
val (database, function) = fun match {
case Some(CatalogAndIdentifierParts(catalog, functionName)) =>
if (isSessionCatalog(catalog)) {
functionName match {
case Seq(db, fn) => (Some(db), Some(fn))
case Seq(fn) => (None, Some(fn))
case _ =>
throw new AnalysisException(s"Unsupported function name '${functionName.quoted}'")
}
} else {
throw new AnalysisException ("SHOW FUNCTIONS is only supported in v1 catalog")
}
case None => (None, pattern)
}
ShowFunctionsCommand(database, function, userScope, systemScope)
}

private def parseV1Table(tableName: Seq[String], sql: String): Seq[String] = {
@@ -236,30 +236,6 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) {
DescribeFunctionCommand(functionName, EXTENDED != null)
}

/**
* Create a plan for a SHOW FUNCTIONS command.
*/
override def visitShowFunctions(ctx: ShowFunctionsContext): LogicalPlan = withOrigin(ctx) {
import ctx._
val (user, system) = Option(ctx.identifier).map(_.getText.toLowerCase(Locale.ROOT)) match {
case None | Some("all") => (true, true)
case Some("system") => (false, true)
case Some("user") => (true, false)
case Some(x) => throw new ParseException(s"SHOW $x FUNCTIONS not supported", ctx)
}

val (db, pat) = if (multipartIdentifier != null) {
val name = visitFunctionName(multipartIdentifier)
(name.database, Some(name.funcName))
} else if (pattern != null) {
(None, Some(string(pattern)))
} else {
(None, None)
}

ShowFunctionsCommand(db, pat, user, system)
}

/**
* Create a [[CreateFunctionCommand]] command.
*
@@ -1799,6 +1799,15 @@ class DataSourceV2SQLSuite
}
}

test("SHOW FUNCTIONS not valid v1 namespace") {
val function = "testcat.ns1.ns2.fun"

val e = intercept[AnalysisException] {
sql(s"SHOW FUNCTIONS LIKE $function")
}
assert(e.message.contains("SHOW FUNCTIONS is only supported in v1 catalog"))
}

test("global temp view should not be masked by v2 catalog") {
val globalTempDB = spark.sessionState.conf.getConf(StaticSQLConf.GLOBAL_TEMP_DATABASE)
spark.conf.set(s"spark.sql.catalog.$globalTempDB", classOf[InMemoryTableCatalog].getName)
@@ -80,21 +80,6 @@ class SparkSqlParserSuite extends AnalysisTest {
intercept("REFRESH", "Resource paths cannot be empty in REFRESH statements")
}

test("show functions") {
assertEqual("show functions", ShowFunctionsCommand(None, None, true, true))
assertEqual("show all functions", ShowFunctionsCommand(None, None, true, true))
assertEqual("show user functions", ShowFunctionsCommand(None, None, true, false))
assertEqual("show system functions", ShowFunctionsCommand(None, None, false, true))
intercept("show special functions", "SHOW special FUNCTIONS")
assertEqual("show functions foo",
ShowFunctionsCommand(None, Some("foo"), true, true))
assertEqual("show functions foo.bar",
ShowFunctionsCommand(Some("foo"), Some("bar"), true, true))
assertEqual("show functions 'foo\\\\.*'",
ShowFunctionsCommand(None, Some("foo\\.*"), true, true))
intercept("show functions foo.bar.baz", "Unsupported function name")
}

test("describe function") {
assertEqual("describe function bar",
DescribeFunctionCommand(FunctionIdentifier("bar", database = None), isExtended = false))
@@ -373,7 +373,7 @@ abstract class HiveComparisonTest

// We will ignore the ExplainCommand, ShowFunctions, DescribeFunction
if ((!hiveQuery.logical.isInstanceOf[ExplainCommand]) &&
(!hiveQuery.logical.isInstanceOf[ShowFunctionsCommand]) &&
(!hiveQuery.logical.isInstanceOf[ShowFunctionsStatement]) &&
(!hiveQuery.logical.isInstanceOf[DescribeFunctionCommand]) &&
(!hiveQuery.logical.isInstanceOf[DescribeCommandBase]) &&
(!hiveQuery.logical.isInstanceOf[DescribeTableStatement]) &&

0 comments on commit bca9de6

Please sign in to comment.
You can’t perform that action at this time.