From 99e5220d0d3f84fd929329a8f2a2a5b5705a5b3b Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Tue, 31 May 2016 22:53:40 +0530 Subject: [PATCH 01/20] [SPARK-15663] SparkSession.catalog.listFunctions shouldn't include the list of built-in functions --- .../org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala index cf9286e6b97a6..461abb0df4491 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala @@ -855,7 +855,8 @@ class SessionCatalog( .map { f => FunctionIdentifier(f, Some(dbName)) } val loadedFunctions = StringUtils.filterPattern(functionRegistry.listFunction(), pattern) .map { f => FunctionIdentifier(f) } - dbFunctions ++ loadedFunctions + val builtInFunctions = FunctionRegistry.builtin.listFunction() + (dbFunctions ++ loadedFunctions).filterNot(builtInFunctions.contains) } From 149bc27f3175d75c94588e6bdce7394aaf52f78e Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Wed, 1 Jun 2016 17:36:54 +0530 Subject: [PATCH 02/20] fix existing tests --- .../apache/spark/sql/catalyst/catalog/SessionCatalog.scala | 4 ++-- .../scala/org/apache/spark/sql/internal/CatalogSuite.scala | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala index 461abb0df4491..0a6308956677d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala @@ -855,8 +855,8 @@ class SessionCatalog( .map { f => FunctionIdentifier(f, Some(dbName)) } val loadedFunctions = StringUtils.filterPattern(functionRegistry.listFunction(), pattern) .map { f => FunctionIdentifier(f) } - val builtInFunctions = FunctionRegistry.builtin.listFunction() - (dbFunctions ++ loadedFunctions).filterNot(builtInFunctions.contains) + val builtInFunctions = FunctionRegistry.builtin.listFunction().toSet + (dbFunctions ++ loadedFunctions).filterNot(f => builtInFunctions.contains(f.funcName)) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala index cd434f7887db6..9957011a075c1 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala @@ -174,8 +174,7 @@ class CatalogSuite } test("list functions") { - assert(Set("+", "current_database", "window").subsetOf( - spark.catalog.listFunctions().collect().map(_.name).toSet)) + assert(spark.catalog.listFunctions().collect().isEmpty) createFunction("my_func1") createFunction("my_func2") createTempFunction("my_temp_func") @@ -192,8 +191,7 @@ class CatalogSuite } test("list functions with database") { - assert(Set("+", "current_database", "window").subsetOf( - spark.catalog.listFunctions("default").collect().map(_.name).toSet)) + assert(spark.catalog.listFunctions("default").collect().isEmpty) createDatabase("my_db1") createDatabase("my_db2") createFunction("my_func1", Some("my_db1")) From 8fe9e377b8139b59a0b280d18111f65ec505b2ed Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Wed, 1 Jun 2016 21:29:08 +0530 Subject: [PATCH 03/20] Fix SQLQuerySuite test --- .../org/apache/spark/sql/SQLQuerySuite.scala | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index 1ddb586d60fd9..ed708ea1e2e75 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -22,6 +22,7 @@ import java.sql.Timestamp import org.apache.spark.AccumulatorSuite import org.apache.spark.sql.catalyst.analysis.UnresolvedException +import org.apache.spark.sql.catalyst.catalog.{CatalogTestUtils, ExternalCatalog, SessionCatalog} import org.apache.spark.sql.catalyst.expressions.SortOrder import org.apache.spark.sql.catalyst.plans.logical.Aggregate import org.apache.spark.sql.catalyst.util.StringUtils @@ -57,16 +58,28 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { } test("show functions") { + val utils = new CatalogTestUtils { + override val tableInputFormat: String = "com.fruit.eyephone.CameraInputFormat" + override val tableOutputFormat: String = "com.fruit.eyephone.CameraOutputFormat" + override def newEmptyCatalog(): ExternalCatalog = spark.sharedState.externalCatalog + } + + def createFunction(names: Seq[String]): Unit = { + names.foreach { name => + spark.sessionState.catalog + .createFunction(utils.newFunc(name, Some("default")), ignoreIfExists = true) + } + } + def getFunctions(pattern: String): Seq[Row] = { - StringUtils.filterPattern(spark.sessionState.functionRegistry.listFunction(), pattern) - .map(Row(_)) + spark.sessionState.catalog.listFunctions("default", pattern).map(Row(_)) } + + createFunction(Seq("logi", "logii", "logiii", "ilog")) + + println(s"getFunctions: ${sql("SHOW functions").collect().toSeq}") // scalastyle:off + println(s"getFunctions: ${getFunctions("*")}") // scalastyle:off checkAnswer(sql("SHOW functions"), getFunctions("*")) - Seq("^c*", "*e$", "log*", "*date*").foreach { pattern => - // For the pattern part, only '*' and '|' are allowed as wildcards. - // For '*', we need to replace it to '.*'. - checkAnswer(sql(s"SHOW FUNCTIONS '$pattern'"), getFunctions(pattern)) - } } test("describe functions") { From b3a67a498dd848ed5361fd68d2ffb18f35b428af Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Thu, 2 Jun 2016 08:37:57 +0530 Subject: [PATCH 04/20] fix show functions test --- .../org/apache/spark/sql/SQLQuerySuite.scala | 20 ++----------------- 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index ed708ea1e2e75..5b6fa4e4eb0d1 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -58,28 +58,12 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { } test("show functions") { - val utils = new CatalogTestUtils { - override val tableInputFormat: String = "com.fruit.eyephone.CameraInputFormat" - override val tableOutputFormat: String = "com.fruit.eyephone.CameraOutputFormat" - override def newEmptyCatalog(): ExternalCatalog = spark.sharedState.externalCatalog - } - - def createFunction(names: Seq[String]): Unit = { - names.foreach { name => - spark.sessionState.catalog - .createFunction(utils.newFunc(name, Some("default")), ignoreIfExists = true) - } - } - def getFunctions(pattern: String): Seq[Row] = { - spark.sessionState.catalog.listFunctions("default", pattern).map(Row(_)) + spark.sessionState.catalog.listFunctions("default").map(Row(_)) } - createFunction(Seq("logi", "logii", "logiii", "ilog")) - - println(s"getFunctions: ${sql("SHOW functions").collect().toSeq}") // scalastyle:off - println(s"getFunctions: ${getFunctions("*")}") // scalastyle:off checkAnswer(sql("SHOW functions"), getFunctions("*")) + assert(sql("SHOW functions").collect().isEmpty) } test("describe functions") { From 5002ab4591e97caa03e5954a2ad548604d419cce Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Thu, 2 Jun 2016 09:41:55 +0530 Subject: [PATCH 05/20] fix show functions test --- .../org/apache/spark/sql/SQLQuerySuite.scala | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index 5b6fa4e4eb0d1..30c0d809a4ddf 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -59,11 +59,32 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { test("show functions") { def getFunctions(pattern: String): Seq[Row] = { - spark.sessionState.catalog.listFunctions("default").map(Row(_)) + StringUtils.filterPattern( + spark.sessionState.catalog.listFunctions("default").map(_.funcName), pattern) + .map(Row(_)) + } + + def createFunction(names: Seq[String]): Unit = { + names.foreach { name => + spark.udf.register(name, (arg1: Int, arg2: String) => arg2 + arg1) + } } - checkAnswer(sql("SHOW functions"), getFunctions("*")) assert(sql("SHOW functions").collect().isEmpty) + + createFunction(Seq("ilog", "logi", "logii", "logiii")) + createFunction(Seq("crc32i", "cubei", "cume_disti")) + createFunction(Seq("isize", "ispace")) + createFunction(Seq("to_datei", "date_addi", "current_datei")) + + checkAnswer(sql("SHOW functions"), getFunctions("*")) + + Seq("^c*", "*e$", "log*", "*date*").foreach { pattern => + // For the pattern part, only '*' and '|' are allowed as wildcards. + // For '*', we need to replace it to '.*'. + checkAnswer(sql(s"SHOW FUNCTIONS '$pattern'"), getFunctions(pattern)) + } + } test("describe functions") { From c3d0067a350c8c42f0d1d394d8d753cfa74331ba Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Thu, 2 Jun 2016 10:27:33 +0530 Subject: [PATCH 06/20] Fix Hive SQLQuerySuite --- .../sql/hive/execution/SQLQuerySuite.scala | 41 +++++++++++-------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index b5691450ca2f0..2f108c422ea4b 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -187,28 +187,35 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { } test("show functions") { - val allBuiltinFunctions = FunctionRegistry.builtin.listFunction().toSet[String].toList.sorted - // The TestContext is shared by all the test cases, some functions may be registered before - // this, so we check that all the builtin functions are returned. - val allFunctions = sql("SHOW functions").collect().map(r => r(0)) - allBuiltinFunctions.foreach { f => - assert(allFunctions.contains(f)) - } withTempDatabase { db => - checkAnswer(sql("SHOW functions abs"), Row("abs")) - checkAnswer(sql("SHOW functions 'abs'"), Row("abs")) - checkAnswer(sql(s"SHOW functions $db.abs"), Row("abs")) - checkAnswer(sql(s"SHOW functions `$db`.`abs`"), Row("abs")) - checkAnswer(sql(s"SHOW functions `$db`.`abs`"), Row("abs")) - checkAnswer(sql("SHOW functions `~`"), Row("~")) + def createFunction(names: Seq[String]): Unit = { + names.foreach { name => + sql( + s""" + |CREATE TEMPORARY FUNCTION $name + |AS '${classOf[PairUDF].getName}' + """.stripMargin) + } + } + createFunction(Seq("temp_abs", "temp_weekofyear", "temp_sha", "temp_sha1", "temp_sha2")) + + checkAnswer(sql("SHOW functions temp_abs"), Row("temp_abs")) + checkAnswer(sql("SHOW functions 'temp_abs'"), Row("temp_abs")) + checkAnswer(sql(s"SHOW functions $db.temp_abs"), Row("temp_abs")) + checkAnswer(sql(s"SHOW functions `$db`.`temp_abs`"), Row("temp_abs")) + checkAnswer(sql(s"SHOW functions `$db`.`temp_abs`"), Row("temp_abs")) checkAnswer(sql("SHOW functions `a function doens't exist`"), Nil) - checkAnswer(sql("SHOW functions `weekofyea*`"), Row("weekofyear")) + checkAnswer(sql("SHOW functions `temp_weekofyea*`"), Row("temp_weekofyear")) + // this probably will failed if we add more function with `sha` prefixing. - checkAnswer(sql("SHOW functions `sha*`"), Row("sha") :: Row("sha1") :: Row("sha2") :: Nil) + checkAnswer( + sql("SHOW functions `temp_sha*`"), + List(Row("temp_sha"), Row("temp_sha1"), Row("temp_sha2"))) + // Test '|' for alternation. checkAnswer( - sql("SHOW functions 'sha*|weekofyea*'"), - Row("sha") :: Row("sha1") :: Row("sha2") :: Row("weekofyear") :: Nil) + sql("SHOW functions 'temp_sha*|temp_weekofyea*'"), + List(Row("temp_sha"), Row("temp_sha1"), Row("temp_sha2"), Row("temp_weekofyear"))) } } From 9755ee7d1f2be4ea3740ff86b80056e2895af3b2 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Thu, 2 Jun 2016 12:11:22 +0530 Subject: [PATCH 07/20] remove failing test --- .../test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 8 -------- 1 file changed, 8 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index c20917b5abbcf..42c67343a42f4 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -106,14 +106,6 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { checkKeywordsExist(sql("describe functioN abcadf"), "Function: abcadf not found.") } - test("SPARK-14415: All functions should have own descriptions") { - for (f <- spark.sessionState.functionRegistry.listFunction()) { - if (!Seq("cube", "grouping", "grouping_id", "rollup", "window").contains(f)) { - checkKeywordsNotExist(sql(s"describe function `$f`"), "N/A.") - } - } - } - test("SPARK-6743: no columns from cache") { Seq( (83, 0, 38), From 5ffd96a07fa7f1d9de5c08f8bfa064764f16fdac Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Thu, 2 Jun 2016 14:49:02 +0530 Subject: [PATCH 08/20] fix sql tests --- python/pyspark/sql/tests.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py index 0d9dd5ea2a364..7ee1b42c53592 100644 --- a/python/pyspark/sql/tests.py +++ b/python/pyspark/sql/tests.py @@ -1482,16 +1482,6 @@ def test_list_functions(self): functions = dict((f.name, f) for f in spark.catalog.listFunctions()) functionsDefault = dict((f.name, f) for f in spark.catalog.listFunctions("default")) self.assertTrue(len(functions) > 200) - self.assertTrue("+" in functions) - self.assertTrue("like" in functions) - self.assertTrue("month" in functions) - self.assertTrue("to_unix_timestamp" in functions) - self.assertTrue("current_database" in functions) - self.assertEquals(functions["+"], Function( - name="+", - description=None, - className="org.apache.spark.sql.catalyst.expressions.Add", - isTemporary=True)) self.assertEquals(functions, functionsDefault) spark.catalog.registerFunction("temp_func", lambda x: str(x)) spark.sql("CREATE FUNCTION func1 AS 'org.apache.spark.data.bricks'") From 1693a4cd01500707b8e5b6f46cf7ef35a0bba22f Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Thu, 2 Jun 2016 16:56:50 +0530 Subject: [PATCH 09/20] fix python sql-test --- python/pyspark/sql/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py index 7ee1b42c53592..e0acde678317d 100644 --- a/python/pyspark/sql/tests.py +++ b/python/pyspark/sql/tests.py @@ -1481,7 +1481,7 @@ def test_list_functions(self): spark.sql("CREATE DATABASE some_db") functions = dict((f.name, f) for f in spark.catalog.listFunctions()) functionsDefault = dict((f.name, f) for f in spark.catalog.listFunctions("default")) - self.assertTrue(len(functions) > 200) + self.assertEquals(len(functions), 0) self.assertEquals(functions, functionsDefault) spark.catalog.registerFunction("temp_func", lambda x: str(x)) spark.sql("CREATE FUNCTION func1 AS 'org.apache.spark.data.bricks'") From 57e952f30ae1976d22adf40ba93d7e19e517c231 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Wed, 8 Jun 2016 09:44:53 +0530 Subject: [PATCH 10/20] remove line --- sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index 55bad93f1b988..201fa35f2b68c 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -84,7 +84,6 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { // For '*', we need to replace it to '.*'. checkAnswer(sql(s"SHOW FUNCTIONS '$pattern'"), getFunctions(pattern)) } - } test("describe functions") { From ec487c5060e37b80b3544f213a10bcee54ae14a8 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Wed, 8 Jun 2016 10:03:25 +0530 Subject: [PATCH 11/20] add test back --- .../test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index 201fa35f2b68c..5077f4b482dd0 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -105,6 +105,14 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { checkKeywordsExist(sql("describe functioN abcadf"), "Function: abcadf not found.") } + test("SPARK-14415: All functions should have own descriptions") { + for (f <- spark.sessionState.functionRegistry.listFunction()) { + if (!Seq("cube", "grouping", "grouping_id", "rollup", "window").contains(f)) { + checkKeywordsNotExist(sql(s"describe function `$f`"), "N/A.") + } + } + } + test("SPARK-6743: no columns from cache") { Seq( (83, 0, 38), From 3c7cd8f6428a9ab7a87d52afee0fc6e549fbe2f7 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Wed, 8 Jun 2016 10:33:41 +0530 Subject: [PATCH 12/20] fix the describe functions test --- .../org/apache/spark/sql/SQLQuerySuite.scala | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index 5077f4b482dd0..75d6b71908d0f 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -21,9 +21,10 @@ import java.math.MathContext import java.sql.Timestamp import org.apache.spark.AccumulatorSuite +import org.apache.spark.sql.catalyst.FunctionIdentifier import org.apache.spark.sql.catalyst.analysis.UnresolvedException import org.apache.spark.sql.catalyst.catalog.{CatalogTestUtils, ExternalCatalog, SessionCatalog} -import org.apache.spark.sql.catalyst.expressions.SortOrder +import org.apache.spark.sql.catalyst.expressions.{ExpressionDescription, SortOrder} import org.apache.spark.sql.catalyst.plans.logical.Aggregate import org.apache.spark.sql.catalyst.util.StringUtils import org.apache.spark.sql.execution.aggregate @@ -70,20 +71,29 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { } } + def dropFunction(names: Seq[String]): Unit = { + names.foreach { name => + spark.sessionState.catalog.dropTempFunction(name, false) + } + } + + val functions = Seq("ilog", "logi", "logii", "logiii", "crc32i", "cubei", "cume_disti", + "isize", "ispace", "to_datei", "date_addi", "current_datei") + assert(sql("SHOW functions").collect().isEmpty) - createFunction(Seq("ilog", "logi", "logii", "logiii")) - createFunction(Seq("crc32i", "cubei", "cume_disti")) - createFunction(Seq("isize", "ispace")) - createFunction(Seq("to_datei", "date_addi", "current_datei")) + createFunction(functions) checkAnswer(sql("SHOW functions"), getFunctions("*")) + assert(sql("SHOW functions").collect().size === functions.size) Seq("^c*", "*e$", "log*", "*date*").foreach { pattern => // For the pattern part, only '*' and '|' are allowed as wildcards. // For '*', we need to replace it to '.*'. checkAnswer(sql(s"SHOW FUNCTIONS '$pattern'"), getFunctions(pattern)) } + + dropFunction(functions) } test("describe functions") { From 5145e533b9d722e1597fd820e2ada776314738ae Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Wed, 8 Jun 2016 10:40:37 +0530 Subject: [PATCH 13/20] remove line --- sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index 75d6b71908d0f..656d3f6256a1e 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -92,7 +92,6 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { // For '*', we need to replace it to '.*'. checkAnswer(sql(s"SHOW FUNCTIONS '$pattern'"), getFunctions(pattern)) } - dropFunction(functions) } From a1ebad4c6205c2ae5d7ba925177d890eb586268b Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Fri, 10 Jun 2016 11:33:22 +0530 Subject: [PATCH 14/20] functionSet instead of listFunction().toSet --- .../apache/spark/sql/catalyst/analysis/FunctionRegistry.scala | 4 ++++ .../apache/spark/sql/catalyst/catalog/SessionCatalog.scala | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala index a7388c71de9e0..c9d8eb1284831 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala @@ -89,6 +89,10 @@ class SimpleFunctionRegistry extends FunctionRegistry { functionBuilders.iterator.map(_._1).toList.sorted } + private[catalyst] def functionSet(): Set[String] = synchronized { + functionBuilders.iterator.map(_._1).toSet + } + override def lookupFunction(name: String): Option[ExpressionInfo] = synchronized { functionBuilders.get(name).map(_._1) } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala index fa9cfa8ecf306..5f5de648fa2c6 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala @@ -855,7 +855,7 @@ class SessionCatalog( .map { f => FunctionIdentifier(f, Some(dbName)) } val loadedFunctions = StringUtils.filterPattern(functionRegistry.listFunction(), pattern) .map { f => FunctionIdentifier(f) } - val builtInFunctions = FunctionRegistry.builtin.listFunction().toSet + val builtInFunctions = FunctionRegistry.builtin.functionSet() (dbFunctions ++ loadedFunctions).filterNot(f => builtInFunctions.contains(f.funcName)) } From fda66c13fb1e85e7876a7c00fbd2d687396ca836 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Fri, 10 Jun 2016 11:35:02 +0530 Subject: [PATCH 15/20] make functionSet lazy --- .../apache/spark/sql/catalyst/analysis/FunctionRegistry.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala index c9d8eb1284831..dde11122fb935 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala @@ -89,7 +89,7 @@ class SimpleFunctionRegistry extends FunctionRegistry { functionBuilders.iterator.map(_._1).toList.sorted } - private[catalyst] def functionSet(): Set[String] = synchronized { + private[catalyst] lazy val functionSet: Set[String] = synchronized { functionBuilders.iterator.map(_._1).toSet } From 4f3cb9925faf3cd830608f1381b53bf7746d5638 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Fri, 10 Jun 2016 11:55:24 +0530 Subject: [PATCH 16/20] make builtInFunctions a set --- .../org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala | 2 +- .../src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala index 5f5de648fa2c6..552106709c3d4 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala @@ -855,7 +855,7 @@ class SessionCatalog( .map { f => FunctionIdentifier(f, Some(dbName)) } val loadedFunctions = StringUtils.filterPattern(functionRegistry.listFunction(), pattern) .map { f => FunctionIdentifier(f) } - val builtInFunctions = FunctionRegistry.builtin.functionSet() + val builtInFunctions = FunctionRegistry.builtin.functionSet (dbFunctions ++ loadedFunctions).filterNot(f => builtInFunctions.contains(f.funcName)) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index 5c1ed1aa1c297..bbe821b76805c 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -77,7 +77,7 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { } } - val functions = Seq("ilog", "logi", "logii", "logiii", "crc32i", "cubei", "cume_disti", + val functions = Array("ilog", "logi", "logii", "logiii", "crc32i", "cubei", "cume_disti", "isize", "ispace", "to_datei", "date_addi", "current_datei") assert(sql("SHOW functions").collect().isEmpty) @@ -86,6 +86,7 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { checkAnswer(sql("SHOW functions"), getFunctions("*")) assert(sql("SHOW functions").collect().size === functions.size) + assert(sql("SHOW functions").collect().toSet === functions.map(Row(_)).toSet) Seq("^c*", "*e$", "log*", "*date*").foreach { pattern => // For the pattern part, only '*' and '|' are allowed as wildcards. From d22668180cec447aae0cf732ebe05f3749ef3b6b Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Fri, 10 Jun 2016 11:56:49 +0530 Subject: [PATCH 17/20] fix style --- .../apache/spark/sql/catalyst/analysis/FunctionRegistry.scala | 2 +- .../apache/spark/sql/catalyst/catalog/SessionCatalog.scala | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala index dde11122fb935..c9d8eb1284831 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala @@ -89,7 +89,7 @@ class SimpleFunctionRegistry extends FunctionRegistry { functionBuilders.iterator.map(_._1).toList.sorted } - private[catalyst] lazy val functionSet: Set[String] = synchronized { + private[catalyst] def functionSet(): Set[String] = synchronized { functionBuilders.iterator.map(_._1).toSet } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala index 552106709c3d4..d963f1aace6eb 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala @@ -855,8 +855,8 @@ class SessionCatalog( .map { f => FunctionIdentifier(f, Some(dbName)) } val loadedFunctions = StringUtils.filterPattern(functionRegistry.listFunction(), pattern) .map { f => FunctionIdentifier(f) } - val builtInFunctions = FunctionRegistry.builtin.functionSet - (dbFunctions ++ loadedFunctions).filterNot(f => builtInFunctions.contains(f.funcName)) + (dbFunctions ++ loadedFunctions) + .filterNot(f => FunctionRegistry.builtin.functionSet.contains(f.funcName)) } From e4aadad19a6f8e64bf8ac58509d29954cb0f6fa4 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Mon, 13 Jun 2016 09:02:21 +0530 Subject: [PATCH 18/20] move functionSet in FunctionRegistry --- .../spark/sql/catalyst/analysis/FunctionRegistry.scala | 9 +++++---- .../spark/sql/catalyst/catalog/SessionCatalog.scala | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala index c9d8eb1284831..473176ee3f478 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala @@ -89,10 +89,6 @@ class SimpleFunctionRegistry extends FunctionRegistry { functionBuilders.iterator.map(_._1).toList.sorted } - private[catalyst] def functionSet(): Set[String] = synchronized { - functionBuilders.iterator.map(_._1).toSet - } - override def lookupFunction(name: String): Option[ExpressionInfo] = synchronized { functionBuilders.get(name).map(_._1) } @@ -400,6 +396,11 @@ object FunctionRegistry { fr } + val functionSet: Set[String] = synchronized { + builtin.listFunction().toSet + } + + /** See usage above. */ private def expression[T <: Expression](name: String) (implicit tag: ClassTag[T]): (String, (ExpressionInfo, FunctionBuilder)) = { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala index d963f1aace6eb..1ec1bb1baf23b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala @@ -856,7 +856,7 @@ class SessionCatalog( val loadedFunctions = StringUtils.filterPattern(functionRegistry.listFunction(), pattern) .map { f => FunctionIdentifier(f) } (dbFunctions ++ loadedFunctions) - .filterNot(f => FunctionRegistry.builtin.functionSet.contains(f.funcName)) + .filterNot(f => FunctionRegistry.functionSet.contains(f.funcName)) } From 41be3ac21501ed1fd6af79a2ea45928834afb24b Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Tue, 14 Jun 2016 00:27:21 +0530 Subject: [PATCH 19/20] remove synchronised from functionSet val --- .../spark/sql/catalyst/analysis/FunctionRegistry.scala | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala index 473176ee3f478..42a8faa412a34 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala @@ -396,10 +396,7 @@ object FunctionRegistry { fr } - val functionSet: Set[String] = synchronized { - builtin.listFunction().toSet - } - + val functionSet: Set[String] = builtin.listFunction().toSet /** See usage above. */ private def expression[T <: Expression](name: String) From 9bfd777298469db49256a52165ba9061dc0f1af3 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Tue, 14 Jun 2016 08:17:08 +0530 Subject: [PATCH 20/20] dropFunctions at the end of test --- .../apache/spark/sql/hive/execution/SQLQuerySuite.scala | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index a1e167f5f899a..1a0eaa66c1391 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -197,6 +197,11 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { """.stripMargin) } } + def dropFunction(names: Seq[String]): Unit = { + names.foreach { name => + sql(s"DROP TEMPORARY FUNCTION $name") + } + } createFunction(Seq("temp_abs", "temp_weekofyear", "temp_sha", "temp_sha1", "temp_sha2")) checkAnswer(sql("SHOW functions temp_abs"), Row("temp_abs")) @@ -216,6 +221,8 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { checkAnswer( sql("SHOW functions 'temp_sha*|temp_weekofyea*'"), List(Row("temp_sha"), Row("temp_sha1"), Row("temp_sha2"), Row("temp_weekofyear"))) + + dropFunction(Seq("temp_abs", "temp_weekofyear", "temp_sha", "temp_sha1", "temp_sha2")) } }