From 066deed940c958ba55ff7dee2d36578a3ea0b8f0 Mon Sep 17 00:00:00 2001 From: Cheolsoo Park Date: Thu, 2 Jul 2015 17:24:00 -0700 Subject: [PATCH 1/3] Type coercion for udf inputs --- .../catalyst/analysis/HiveTypeCoercion.scala | 2 +- .../sql/catalyst/expressions/ScalaUDF.scala | 9 ++- .../apache/spark/sql/UDFRegistration.scala | 69 ++++++++++++------- .../spark/sql/UserDefinedFunction.scala | 7 +- .../org/apache/spark/sql/functions.scala | 33 ++++++--- .../scala/org/apache/spark/sql/UDFSuite.scala | 6 ++ 6 files changed, 87 insertions(+), 39 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala index 0bc893224026e..b18887abe317a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala @@ -711,7 +711,7 @@ object HiveTypeCoercion { // Skip nodes who's children have not been resolved yet. case e if !e.childrenResolved => e - case e: ExpectsInputTypes => + case e: ExpectsInputTypes if (e.inputTypes.nonEmpty) => val children: Seq[Expression] = e.children.zip(e.inputTypes).map { case (in, expected) => implicitCast(in, expected) } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala index caf021b016a41..fa782a3ce83cc 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala @@ -24,13 +24,18 @@ import org.apache.spark.sql.types.DataType * User-defined function. * @param dataType Return type of function. */ -case class ScalaUDF(function: AnyRef, dataType: DataType, children: Seq[Expression]) - extends Expression { +case class ScalaUDF( + function: AnyRef, + dataType: DataType, + children: Seq[Expression], + expectedInputTypes: Seq[DataType] = Nil) extends Expression with ExpectsInputTypes { override def nullable: Boolean = true override def toString: String = s"UDF(${children.mkString(",")})" + override def inputTypes: Seq[DataType] = expectedInputTypes + // scalastyle:off /** This method has been generated by this script diff --git a/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala b/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala index 03dc37aa73f0c..a4bcf78a18e44 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala @@ -126,7 +126,8 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag](name: String, func: Function0[RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e) + val inputTypes = Seq[DataType]() + def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) } @@ -138,7 +139,8 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag](name: String, func: Function1[A1, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType) + def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) } @@ -150,7 +152,8 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag](name: String, func: Function2[A1, A2, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType) + def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) } @@ -162,7 +165,8 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag](name: String, func: Function3[A1, A2, A3, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType) + def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) } @@ -174,7 +178,8 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag](name: String, func: Function4[A1, A2, A3, A4, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType) + def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) } @@ -186,7 +191,8 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag](name: String, func: Function5[A1, A2, A3, A4, A5, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType) + def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) } @@ -198,7 +204,8 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag](name: String, func: Function6[A1, A2, A3, A4, A5, A6, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType) + def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) } @@ -210,7 +217,8 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag](name: String, func: Function7[A1, A2, A3, A4, A5, A6, A7, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType) + def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) } @@ -222,7 +230,8 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag](name: String, func: Function8[A1, A2, A3, A4, A5, A6, A7, A8, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType) + def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) } @@ -234,7 +243,8 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag](name: String, func: Function9[A1, A2, A3, A4, A5, A6, A7, A8, A9, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType) + def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) } @@ -246,7 +256,8 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag](name: String, func: Function10[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType, ScalaReflection.schemaFor[A10].dataType) + def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) } @@ -258,7 +269,8 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag](name: String, func: Function11[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType, ScalaReflection.schemaFor[A10].dataType, ScalaReflection.schemaFor[A11].dataType) + def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) } @@ -270,7 +282,8 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag](name: String, func: Function12[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType, ScalaReflection.schemaFor[A10].dataType, ScalaReflection.schemaFor[A11].dataType, ScalaReflection.schemaFor[A12].dataType) + def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) } @@ -282,7 +295,8 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag](name: String, func: Function13[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType, ScalaReflection.schemaFor[A10].dataType, ScalaReflection.schemaFor[A11].dataType, ScalaReflection.schemaFor[A12].dataType, ScalaReflection.schemaFor[A13].dataType) + def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) } @@ -294,7 +308,8 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag](name: String, func: Function14[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType, ScalaReflection.schemaFor[A10].dataType, ScalaReflection.schemaFor[A11].dataType, ScalaReflection.schemaFor[A12].dataType, ScalaReflection.schemaFor[A13].dataType, ScalaReflection.schemaFor[A14].dataType) + def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) } @@ -306,7 +321,8 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag](name: String, func: Function15[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType, ScalaReflection.schemaFor[A10].dataType, ScalaReflection.schemaFor[A11].dataType, ScalaReflection.schemaFor[A12].dataType, ScalaReflection.schemaFor[A13].dataType, ScalaReflection.schemaFor[A14].dataType, ScalaReflection.schemaFor[A15].dataType) + def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) } @@ -318,7 +334,8 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag](name: String, func: Function16[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType, ScalaReflection.schemaFor[A10].dataType, ScalaReflection.schemaFor[A11].dataType, ScalaReflection.schemaFor[A12].dataType, ScalaReflection.schemaFor[A13].dataType, ScalaReflection.schemaFor[A14].dataType, ScalaReflection.schemaFor[A15].dataType, ScalaReflection.schemaFor[A16].dataType) + def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) } @@ -330,7 +347,8 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag](name: String, func: Function17[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType, ScalaReflection.schemaFor[A10].dataType, ScalaReflection.schemaFor[A11].dataType, ScalaReflection.schemaFor[A12].dataType, ScalaReflection.schemaFor[A13].dataType, ScalaReflection.schemaFor[A14].dataType, ScalaReflection.schemaFor[A15].dataType, ScalaReflection.schemaFor[A16].dataType, ScalaReflection.schemaFor[A17].dataType) + def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) } @@ -342,7 +360,8 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag](name: String, func: Function18[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType, ScalaReflection.schemaFor[A10].dataType, ScalaReflection.schemaFor[A11].dataType, ScalaReflection.schemaFor[A12].dataType, ScalaReflection.schemaFor[A13].dataType, ScalaReflection.schemaFor[A14].dataType, ScalaReflection.schemaFor[A15].dataType, ScalaReflection.schemaFor[A16].dataType, ScalaReflection.schemaFor[A17].dataType, ScalaReflection.schemaFor[A18].dataType) + def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) } @@ -354,7 +373,8 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag](name: String, func: Function19[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType, ScalaReflection.schemaFor[A10].dataType, ScalaReflection.schemaFor[A11].dataType, ScalaReflection.schemaFor[A12].dataType, ScalaReflection.schemaFor[A13].dataType, ScalaReflection.schemaFor[A14].dataType, ScalaReflection.schemaFor[A15].dataType, ScalaReflection.schemaFor[A16].dataType, ScalaReflection.schemaFor[A17].dataType, ScalaReflection.schemaFor[A18].dataType, ScalaReflection.schemaFor[A19].dataType) + def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) } @@ -366,7 +386,8 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag](name: String, func: Function20[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType, ScalaReflection.schemaFor[A10].dataType, ScalaReflection.schemaFor[A11].dataType, ScalaReflection.schemaFor[A12].dataType, ScalaReflection.schemaFor[A13].dataType, ScalaReflection.schemaFor[A14].dataType, ScalaReflection.schemaFor[A15].dataType, ScalaReflection.schemaFor[A16].dataType, ScalaReflection.schemaFor[A17].dataType, ScalaReflection.schemaFor[A18].dataType, ScalaReflection.schemaFor[A19].dataType, ScalaReflection.schemaFor[A20].dataType) + def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) } @@ -378,7 +399,8 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag, A21: TypeTag](name: String, func: Function21[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, A21, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType, ScalaReflection.schemaFor[A10].dataType, ScalaReflection.schemaFor[A11].dataType, ScalaReflection.schemaFor[A12].dataType, ScalaReflection.schemaFor[A13].dataType, ScalaReflection.schemaFor[A14].dataType, ScalaReflection.schemaFor[A15].dataType, ScalaReflection.schemaFor[A16].dataType, ScalaReflection.schemaFor[A17].dataType, ScalaReflection.schemaFor[A18].dataType, ScalaReflection.schemaFor[A19].dataType, ScalaReflection.schemaFor[A20].dataType, ScalaReflection.schemaFor[A21].dataType) + def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) } @@ -390,7 +412,8 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag, A21: TypeTag, A22: TypeTag](name: String, func: Function22[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, A21, A22, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType, ScalaReflection.schemaFor[A10].dataType, ScalaReflection.schemaFor[A11].dataType, ScalaReflection.schemaFor[A12].dataType, ScalaReflection.schemaFor[A13].dataType, ScalaReflection.schemaFor[A14].dataType, ScalaReflection.schemaFor[A15].dataType, ScalaReflection.schemaFor[A16].dataType, ScalaReflection.schemaFor[A17].dataType, ScalaReflection.schemaFor[A18].dataType, ScalaReflection.schemaFor[A19].dataType, ScalaReflection.schemaFor[A20].dataType, ScalaReflection.schemaFor[A21].dataType, ScalaReflection.schemaFor[A22].dataType) + def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/UserDefinedFunction.scala b/sql/core/src/main/scala/org/apache/spark/sql/UserDefinedFunction.scala index 831eb7eb0fae9..b14e00ab9b163 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/UserDefinedFunction.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/UserDefinedFunction.scala @@ -41,10 +41,13 @@ import org.apache.spark.sql.types.DataType * @since 1.3.0 */ @Experimental -case class UserDefinedFunction protected[sql] (f: AnyRef, dataType: DataType) { +case class UserDefinedFunction protected[sql] ( + f: AnyRef, + dataType: DataType, + inputTypes: Seq[DataType] = Nil) { def apply(exprs: Column*): Column = { - Column(ScalaUDF(f, dataType, exprs.map(_.expr))) + Column(ScalaUDF(f, dataType, exprs.map(_.expr), inputTypes)) } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala index 4ee1fb8374b07..c3cfcaed1ee27 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala @@ -1589,7 +1589,8 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag](f: Function0[RT]): UserDefinedFunction = { - UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType) + val inputTypes = Seq[DataType]() + UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } /** @@ -1600,7 +1601,8 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag, A1: TypeTag](f: Function1[A1, RT]): UserDefinedFunction = { - UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType) + UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } /** @@ -1611,7 +1613,8 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag](f: Function2[A1, A2, RT]): UserDefinedFunction = { - UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType) + UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } /** @@ -1622,7 +1625,8 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag](f: Function3[A1, A2, A3, RT]): UserDefinedFunction = { - UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType) + UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } /** @@ -1633,7 +1637,8 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag](f: Function4[A1, A2, A3, A4, RT]): UserDefinedFunction = { - UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType) + UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } /** @@ -1644,7 +1649,8 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag](f: Function5[A1, A2, A3, A4, A5, RT]): UserDefinedFunction = { - UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType) + UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } /** @@ -1655,7 +1661,8 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag](f: Function6[A1, A2, A3, A4, A5, A6, RT]): UserDefinedFunction = { - UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType) + UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } /** @@ -1666,7 +1673,8 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag](f: Function7[A1, A2, A3, A4, A5, A6, A7, RT]): UserDefinedFunction = { - UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType) + UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } /** @@ -1677,7 +1685,8 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag](f: Function8[A1, A2, A3, A4, A5, A6, A7, A8, RT]): UserDefinedFunction = { - UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType) + UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } /** @@ -1688,7 +1697,8 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag](f: Function9[A1, A2, A3, A4, A5, A6, A7, A8, A9, RT]): UserDefinedFunction = { - UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType) + UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } /** @@ -1699,7 +1709,8 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag](f: Function10[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, RT]): UserDefinedFunction = { - UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType) + val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType, ScalaReflection.schemaFor[A10].dataType) + UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } ////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala index 703a34c47ec20..b8d3e38d99ebf 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala @@ -96,4 +96,10 @@ class UDFSuite extends QueryTest { // 1 + 1 is constant folded causing a transformation. assert(ctx.sql("SELECT makeStruct(1 + 1, 2)").first().getAs[Row](0) === Row(2, 2)) } + + test("type coercion for udf inputs") { + ctx.udf.register("intExpected", (x: Int) => x) + // pass a decimal to intExpected. + assert(ctx.sql("SELECT intExpected(1.0)").head().getInt(0) === 1) + } } From dce1efd4f50e9ba0f356388c04e5bfddc8fc701c Mon Sep 17 00:00:00 2001 From: Cheolsoo Park Date: Thu, 2 Jul 2015 22:27:37 -0700 Subject: [PATCH 2/3] Fix unit tests and update the codegen script --- .../apache/spark/sql/UDFRegistration.scala | 52 ++++++++++--------- .../org/apache/spark/sql/functions.scala | 27 +++++----- 2 files changed, 42 insertions(+), 37 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala b/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala index a4bcf78a18e44..e916e0776fe04 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala @@ -20,6 +20,7 @@ package org.apache.spark.sql import java.util.{List => JList, Map => JMap} import scala.reflect.runtime.universe.TypeTag +import scala.util.Try import org.apache.spark.{Accumulator, Logging} import org.apache.spark.api.python.PythonBroadcast @@ -30,7 +31,6 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ScalaUDF} import org.apache.spark.sql.execution.PythonUDF import org.apache.spark.sql.types.DataType - /** * Functions for registering user-defined functions. Use [[SQLContext.udf]] to access this. * @@ -87,6 +87,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { (0 to 22).map { x => val types = (1 to x).foldRight("RT")((i, s) => {s"A$i, $s"}) val typeTags = (1 to x).map(i => s"A${i}: TypeTag").foldLeft("RT: TypeTag")(_ + ", " + _) + val inputTypes = (1 to x).foldLeft("Nil")((s, i) => {s"$s :+ ScalaReflection.schemaFor[A$i].dataType"}) println(s""" /** * Register a Scala closure of ${x} arguments as user-defined function (UDF). @@ -95,7 +96,8 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[$typeTags](name: String, func: Function$x[$types]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e) + val inputTypes = Try($inputTypes).getOrElse(Nil) + def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) }""") @@ -126,7 +128,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag](name: String, func: Function0[RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Seq[DataType]() + val inputTypes = Try(Nil).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -139,7 +141,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag](name: String, func: Function1[A1, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -152,7 +154,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag](name: String, func: Function2[A1, A2, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -165,7 +167,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag](name: String, func: Function3[A1, A2, A3, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -178,7 +180,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag](name: String, func: Function4[A1, A2, A3, A4, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -191,7 +193,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag](name: String, func: Function5[A1, A2, A3, A4, A5, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -204,7 +206,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag](name: String, func: Function6[A1, A2, A3, A4, A5, A6, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -217,7 +219,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag](name: String, func: Function7[A1, A2, A3, A4, A5, A6, A7, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -230,7 +232,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag](name: String, func: Function8[A1, A2, A3, A4, A5, A6, A7, A8, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType :+ ScalaReflection.schemaFor[A8].dataType).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -243,7 +245,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag](name: String, func: Function9[A1, A2, A3, A4, A5, A6, A7, A8, A9, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType :+ ScalaReflection.schemaFor[A8].dataType :+ ScalaReflection.schemaFor[A9].dataType).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -256,7 +258,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag](name: String, func: Function10[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType, ScalaReflection.schemaFor[A10].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType :+ ScalaReflection.schemaFor[A8].dataType :+ ScalaReflection.schemaFor[A9].dataType :+ ScalaReflection.schemaFor[A10].dataType).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -269,7 +271,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag](name: String, func: Function11[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType, ScalaReflection.schemaFor[A10].dataType, ScalaReflection.schemaFor[A11].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType :+ ScalaReflection.schemaFor[A8].dataType :+ ScalaReflection.schemaFor[A9].dataType :+ ScalaReflection.schemaFor[A10].dataType :+ ScalaReflection.schemaFor[A11].dataType).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -282,7 +284,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag](name: String, func: Function12[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType, ScalaReflection.schemaFor[A10].dataType, ScalaReflection.schemaFor[A11].dataType, ScalaReflection.schemaFor[A12].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType :+ ScalaReflection.schemaFor[A8].dataType :+ ScalaReflection.schemaFor[A9].dataType :+ ScalaReflection.schemaFor[A10].dataType :+ ScalaReflection.schemaFor[A11].dataType :+ ScalaReflection.schemaFor[A12].dataType).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -295,7 +297,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag](name: String, func: Function13[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType, ScalaReflection.schemaFor[A10].dataType, ScalaReflection.schemaFor[A11].dataType, ScalaReflection.schemaFor[A12].dataType, ScalaReflection.schemaFor[A13].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType :+ ScalaReflection.schemaFor[A8].dataType :+ ScalaReflection.schemaFor[A9].dataType :+ ScalaReflection.schemaFor[A10].dataType :+ ScalaReflection.schemaFor[A11].dataType :+ ScalaReflection.schemaFor[A12].dataType :+ ScalaReflection.schemaFor[A13].dataType).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -308,7 +310,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag](name: String, func: Function14[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType, ScalaReflection.schemaFor[A10].dataType, ScalaReflection.schemaFor[A11].dataType, ScalaReflection.schemaFor[A12].dataType, ScalaReflection.schemaFor[A13].dataType, ScalaReflection.schemaFor[A14].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType :+ ScalaReflection.schemaFor[A8].dataType :+ ScalaReflection.schemaFor[A9].dataType :+ ScalaReflection.schemaFor[A10].dataType :+ ScalaReflection.schemaFor[A11].dataType :+ ScalaReflection.schemaFor[A12].dataType :+ ScalaReflection.schemaFor[A13].dataType :+ ScalaReflection.schemaFor[A14].dataType).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -321,7 +323,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag](name: String, func: Function15[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType, ScalaReflection.schemaFor[A10].dataType, ScalaReflection.schemaFor[A11].dataType, ScalaReflection.schemaFor[A12].dataType, ScalaReflection.schemaFor[A13].dataType, ScalaReflection.schemaFor[A14].dataType, ScalaReflection.schemaFor[A15].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType :+ ScalaReflection.schemaFor[A8].dataType :+ ScalaReflection.schemaFor[A9].dataType :+ ScalaReflection.schemaFor[A10].dataType :+ ScalaReflection.schemaFor[A11].dataType :+ ScalaReflection.schemaFor[A12].dataType :+ ScalaReflection.schemaFor[A13].dataType :+ ScalaReflection.schemaFor[A14].dataType :+ ScalaReflection.schemaFor[A15].dataType).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -334,7 +336,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag](name: String, func: Function16[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType, ScalaReflection.schemaFor[A10].dataType, ScalaReflection.schemaFor[A11].dataType, ScalaReflection.schemaFor[A12].dataType, ScalaReflection.schemaFor[A13].dataType, ScalaReflection.schemaFor[A14].dataType, ScalaReflection.schemaFor[A15].dataType, ScalaReflection.schemaFor[A16].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType :+ ScalaReflection.schemaFor[A8].dataType :+ ScalaReflection.schemaFor[A9].dataType :+ ScalaReflection.schemaFor[A10].dataType :+ ScalaReflection.schemaFor[A11].dataType :+ ScalaReflection.schemaFor[A12].dataType :+ ScalaReflection.schemaFor[A13].dataType :+ ScalaReflection.schemaFor[A14].dataType :+ ScalaReflection.schemaFor[A15].dataType :+ ScalaReflection.schemaFor[A16].dataType).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -347,7 +349,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag](name: String, func: Function17[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType, ScalaReflection.schemaFor[A10].dataType, ScalaReflection.schemaFor[A11].dataType, ScalaReflection.schemaFor[A12].dataType, ScalaReflection.schemaFor[A13].dataType, ScalaReflection.schemaFor[A14].dataType, ScalaReflection.schemaFor[A15].dataType, ScalaReflection.schemaFor[A16].dataType, ScalaReflection.schemaFor[A17].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType :+ ScalaReflection.schemaFor[A8].dataType :+ ScalaReflection.schemaFor[A9].dataType :+ ScalaReflection.schemaFor[A10].dataType :+ ScalaReflection.schemaFor[A11].dataType :+ ScalaReflection.schemaFor[A12].dataType :+ ScalaReflection.schemaFor[A13].dataType :+ ScalaReflection.schemaFor[A14].dataType :+ ScalaReflection.schemaFor[A15].dataType :+ ScalaReflection.schemaFor[A16].dataType :+ ScalaReflection.schemaFor[A17].dataType).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -360,7 +362,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag](name: String, func: Function18[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType, ScalaReflection.schemaFor[A10].dataType, ScalaReflection.schemaFor[A11].dataType, ScalaReflection.schemaFor[A12].dataType, ScalaReflection.schemaFor[A13].dataType, ScalaReflection.schemaFor[A14].dataType, ScalaReflection.schemaFor[A15].dataType, ScalaReflection.schemaFor[A16].dataType, ScalaReflection.schemaFor[A17].dataType, ScalaReflection.schemaFor[A18].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType :+ ScalaReflection.schemaFor[A8].dataType :+ ScalaReflection.schemaFor[A9].dataType :+ ScalaReflection.schemaFor[A10].dataType :+ ScalaReflection.schemaFor[A11].dataType :+ ScalaReflection.schemaFor[A12].dataType :+ ScalaReflection.schemaFor[A13].dataType :+ ScalaReflection.schemaFor[A14].dataType :+ ScalaReflection.schemaFor[A15].dataType :+ ScalaReflection.schemaFor[A16].dataType :+ ScalaReflection.schemaFor[A17].dataType :+ ScalaReflection.schemaFor[A18].dataType).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -373,7 +375,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag](name: String, func: Function19[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType, ScalaReflection.schemaFor[A10].dataType, ScalaReflection.schemaFor[A11].dataType, ScalaReflection.schemaFor[A12].dataType, ScalaReflection.schemaFor[A13].dataType, ScalaReflection.schemaFor[A14].dataType, ScalaReflection.schemaFor[A15].dataType, ScalaReflection.schemaFor[A16].dataType, ScalaReflection.schemaFor[A17].dataType, ScalaReflection.schemaFor[A18].dataType, ScalaReflection.schemaFor[A19].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType :+ ScalaReflection.schemaFor[A8].dataType :+ ScalaReflection.schemaFor[A9].dataType :+ ScalaReflection.schemaFor[A10].dataType :+ ScalaReflection.schemaFor[A11].dataType :+ ScalaReflection.schemaFor[A12].dataType :+ ScalaReflection.schemaFor[A13].dataType :+ ScalaReflection.schemaFor[A14].dataType :+ ScalaReflection.schemaFor[A15].dataType :+ ScalaReflection.schemaFor[A16].dataType :+ ScalaReflection.schemaFor[A17].dataType :+ ScalaReflection.schemaFor[A18].dataType :+ ScalaReflection.schemaFor[A19].dataType).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -386,7 +388,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag](name: String, func: Function20[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType, ScalaReflection.schemaFor[A10].dataType, ScalaReflection.schemaFor[A11].dataType, ScalaReflection.schemaFor[A12].dataType, ScalaReflection.schemaFor[A13].dataType, ScalaReflection.schemaFor[A14].dataType, ScalaReflection.schemaFor[A15].dataType, ScalaReflection.schemaFor[A16].dataType, ScalaReflection.schemaFor[A17].dataType, ScalaReflection.schemaFor[A18].dataType, ScalaReflection.schemaFor[A19].dataType, ScalaReflection.schemaFor[A20].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType :+ ScalaReflection.schemaFor[A8].dataType :+ ScalaReflection.schemaFor[A9].dataType :+ ScalaReflection.schemaFor[A10].dataType :+ ScalaReflection.schemaFor[A11].dataType :+ ScalaReflection.schemaFor[A12].dataType :+ ScalaReflection.schemaFor[A13].dataType :+ ScalaReflection.schemaFor[A14].dataType :+ ScalaReflection.schemaFor[A15].dataType :+ ScalaReflection.schemaFor[A16].dataType :+ ScalaReflection.schemaFor[A17].dataType :+ ScalaReflection.schemaFor[A18].dataType :+ ScalaReflection.schemaFor[A19].dataType :+ ScalaReflection.schemaFor[A20].dataType).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -399,7 +401,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag, A21: TypeTag](name: String, func: Function21[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, A21, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType, ScalaReflection.schemaFor[A10].dataType, ScalaReflection.schemaFor[A11].dataType, ScalaReflection.schemaFor[A12].dataType, ScalaReflection.schemaFor[A13].dataType, ScalaReflection.schemaFor[A14].dataType, ScalaReflection.schemaFor[A15].dataType, ScalaReflection.schemaFor[A16].dataType, ScalaReflection.schemaFor[A17].dataType, ScalaReflection.schemaFor[A18].dataType, ScalaReflection.schemaFor[A19].dataType, ScalaReflection.schemaFor[A20].dataType, ScalaReflection.schemaFor[A21].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType :+ ScalaReflection.schemaFor[A8].dataType :+ ScalaReflection.schemaFor[A9].dataType :+ ScalaReflection.schemaFor[A10].dataType :+ ScalaReflection.schemaFor[A11].dataType :+ ScalaReflection.schemaFor[A12].dataType :+ ScalaReflection.schemaFor[A13].dataType :+ ScalaReflection.schemaFor[A14].dataType :+ ScalaReflection.schemaFor[A15].dataType :+ ScalaReflection.schemaFor[A16].dataType :+ ScalaReflection.schemaFor[A17].dataType :+ ScalaReflection.schemaFor[A18].dataType :+ ScalaReflection.schemaFor[A19].dataType :+ ScalaReflection.schemaFor[A20].dataType :+ ScalaReflection.schemaFor[A21].dataType).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -412,7 +414,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag, A21: TypeTag, A22: TypeTag](name: String, func: Function22[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, A21, A22, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType, ScalaReflection.schemaFor[A10].dataType, ScalaReflection.schemaFor[A11].dataType, ScalaReflection.schemaFor[A12].dataType, ScalaReflection.schemaFor[A13].dataType, ScalaReflection.schemaFor[A14].dataType, ScalaReflection.schemaFor[A15].dataType, ScalaReflection.schemaFor[A16].dataType, ScalaReflection.schemaFor[A17].dataType, ScalaReflection.schemaFor[A18].dataType, ScalaReflection.schemaFor[A19].dataType, ScalaReflection.schemaFor[A20].dataType, ScalaReflection.schemaFor[A21].dataType, ScalaReflection.schemaFor[A22].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType :+ ScalaReflection.schemaFor[A8].dataType :+ ScalaReflection.schemaFor[A9].dataType :+ ScalaReflection.schemaFor[A10].dataType :+ ScalaReflection.schemaFor[A11].dataType :+ ScalaReflection.schemaFor[A12].dataType :+ ScalaReflection.schemaFor[A13].dataType :+ ScalaReflection.schemaFor[A14].dataType :+ ScalaReflection.schemaFor[A15].dataType :+ ScalaReflection.schemaFor[A16].dataType :+ ScalaReflection.schemaFor[A17].dataType :+ ScalaReflection.schemaFor[A18].dataType :+ ScalaReflection.schemaFor[A19].dataType :+ ScalaReflection.schemaFor[A20].dataType :+ ScalaReflection.schemaFor[A21].dataType :+ ScalaReflection.schemaFor[A22].dataType).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala index c3cfcaed1ee27..57f7f5ff55a8d 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql import scala.language.implicitConversions import scala.reflect.runtime.universe.{TypeTag, typeTag} +import scala.util.Try import org.apache.spark.annotation.Experimental import org.apache.spark.sql.catalyst.ScalaReflection @@ -1548,6 +1549,7 @@ object functions { (0 to 10).map { x => val types = (1 to x).foldRight("RT")((i, s) => {s"A$i, $s"}) val typeTags = (1 to x).map(i => s"A$i: TypeTag").foldLeft("RT: TypeTag")(_ + ", " + _) + val inputTypes = (1 to x).foldLeft("Nil")((s, i) => {s"$s :+ ScalaReflection.schemaFor(typeTag[A$i]).dataType"}) println(s""" /** * Defines a user-defined function of ${x} arguments as user-defined function (UDF). @@ -1557,7 +1559,8 @@ object functions { * @since 1.3.0 */ def udf[$typeTags](f: Function$x[$types]): UserDefinedFunction = { - UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType) + val inputTypes = Try($inputTypes).getOrElse(Nil) + UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) }""") } @@ -1589,7 +1592,7 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag](f: Function0[RT]): UserDefinedFunction = { - val inputTypes = Seq[DataType]() + val inputTypes = Try(Nil).getOrElse(Nil) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } @@ -1601,7 +1604,7 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag, A1: TypeTag](f: Function1[A1, RT]): UserDefinedFunction = { - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor(typeTag[A1]).dataType).getOrElse(Nil) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } @@ -1613,7 +1616,7 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag](f: Function2[A1, A2, RT]): UserDefinedFunction = { - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor(typeTag[A1]).dataType :+ ScalaReflection.schemaFor(typeTag[A2]).dataType).getOrElse(Nil) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } @@ -1625,7 +1628,7 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag](f: Function3[A1, A2, A3, RT]): UserDefinedFunction = { - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor(typeTag[A1]).dataType :+ ScalaReflection.schemaFor(typeTag[A2]).dataType :+ ScalaReflection.schemaFor(typeTag[A3]).dataType).getOrElse(Nil) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } @@ -1637,7 +1640,7 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag](f: Function4[A1, A2, A3, A4, RT]): UserDefinedFunction = { - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor(typeTag[A1]).dataType :+ ScalaReflection.schemaFor(typeTag[A2]).dataType :+ ScalaReflection.schemaFor(typeTag[A3]).dataType :+ ScalaReflection.schemaFor(typeTag[A4]).dataType).getOrElse(Nil) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } @@ -1649,7 +1652,7 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag](f: Function5[A1, A2, A3, A4, A5, RT]): UserDefinedFunction = { - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor(typeTag[A1]).dataType :+ ScalaReflection.schemaFor(typeTag[A2]).dataType :+ ScalaReflection.schemaFor(typeTag[A3]).dataType :+ ScalaReflection.schemaFor(typeTag[A4]).dataType :+ ScalaReflection.schemaFor(typeTag[A5]).dataType).getOrElse(Nil) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } @@ -1661,7 +1664,7 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag](f: Function6[A1, A2, A3, A4, A5, A6, RT]): UserDefinedFunction = { - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor(typeTag[A1]).dataType :+ ScalaReflection.schemaFor(typeTag[A2]).dataType :+ ScalaReflection.schemaFor(typeTag[A3]).dataType :+ ScalaReflection.schemaFor(typeTag[A4]).dataType :+ ScalaReflection.schemaFor(typeTag[A5]).dataType :+ ScalaReflection.schemaFor(typeTag[A6]).dataType).getOrElse(Nil) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } @@ -1673,7 +1676,7 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag](f: Function7[A1, A2, A3, A4, A5, A6, A7, RT]): UserDefinedFunction = { - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor(typeTag[A1]).dataType :+ ScalaReflection.schemaFor(typeTag[A2]).dataType :+ ScalaReflection.schemaFor(typeTag[A3]).dataType :+ ScalaReflection.schemaFor(typeTag[A4]).dataType :+ ScalaReflection.schemaFor(typeTag[A5]).dataType :+ ScalaReflection.schemaFor(typeTag[A6]).dataType :+ ScalaReflection.schemaFor(typeTag[A7]).dataType).getOrElse(Nil) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } @@ -1685,7 +1688,7 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag](f: Function8[A1, A2, A3, A4, A5, A6, A7, A8, RT]): UserDefinedFunction = { - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor(typeTag[A1]).dataType :+ ScalaReflection.schemaFor(typeTag[A2]).dataType :+ ScalaReflection.schemaFor(typeTag[A3]).dataType :+ ScalaReflection.schemaFor(typeTag[A4]).dataType :+ ScalaReflection.schemaFor(typeTag[A5]).dataType :+ ScalaReflection.schemaFor(typeTag[A6]).dataType :+ ScalaReflection.schemaFor(typeTag[A7]).dataType :+ ScalaReflection.schemaFor(typeTag[A8]).dataType).getOrElse(Nil) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } @@ -1697,7 +1700,7 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag](f: Function9[A1, A2, A3, A4, A5, A6, A7, A8, A9, RT]): UserDefinedFunction = { - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor(typeTag[A1]).dataType :+ ScalaReflection.schemaFor(typeTag[A2]).dataType :+ ScalaReflection.schemaFor(typeTag[A3]).dataType :+ ScalaReflection.schemaFor(typeTag[A4]).dataType :+ ScalaReflection.schemaFor(typeTag[A5]).dataType :+ ScalaReflection.schemaFor(typeTag[A6]).dataType :+ ScalaReflection.schemaFor(typeTag[A7]).dataType :+ ScalaReflection.schemaFor(typeTag[A8]).dataType :+ ScalaReflection.schemaFor(typeTag[A9]).dataType).getOrElse(Nil) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } @@ -1709,7 +1712,7 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag](f: Function10[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, RT]): UserDefinedFunction = { - val inputTypes = Seq[DataType](ScalaReflection.schemaFor[A1].dataType, ScalaReflection.schemaFor[A2].dataType, ScalaReflection.schemaFor[A3].dataType, ScalaReflection.schemaFor[A4].dataType, ScalaReflection.schemaFor[A5].dataType, ScalaReflection.schemaFor[A6].dataType, ScalaReflection.schemaFor[A7].dataType, ScalaReflection.schemaFor[A8].dataType, ScalaReflection.schemaFor[A9].dataType, ScalaReflection.schemaFor[A10].dataType) + val inputTypes = Try(Nil :+ ScalaReflection.schemaFor(typeTag[A1]).dataType :+ ScalaReflection.schemaFor(typeTag[A2]).dataType :+ ScalaReflection.schemaFor(typeTag[A3]).dataType :+ ScalaReflection.schemaFor(typeTag[A4]).dataType :+ ScalaReflection.schemaFor(typeTag[A5]).dataType :+ ScalaReflection.schemaFor(typeTag[A6]).dataType :+ ScalaReflection.schemaFor(typeTag[A7]).dataType :+ ScalaReflection.schemaFor(typeTag[A8]).dataType :+ ScalaReflection.schemaFor(typeTag[A9]).dataType :+ ScalaReflection.schemaFor(typeTag[A10]).dataType).getOrElse(Nil) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } From 2d0ed1578589adf8ad3cbf4bfaff085fb27171df Mon Sep 17 00:00:00 2001 From: Cheolsoo Park Date: Thu, 2 Jul 2015 23:14:31 -0700 Subject: [PATCH 3/3] Incorporate comments --- .../sql/catalyst/expressions/ScalaUDF.scala | 4 +- .../apache/spark/sql/UDFRegistration.scala | 46 +++++++++---------- .../org/apache/spark/sql/functions.scala | 22 ++++----- 3 files changed, 35 insertions(+), 37 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala index fa782a3ce83cc..fc055c97a179f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala @@ -28,14 +28,12 @@ case class ScalaUDF( function: AnyRef, dataType: DataType, children: Seq[Expression], - expectedInputTypes: Seq[DataType] = Nil) extends Expression with ExpectsInputTypes { + inputTypes: Seq[DataType] = Nil) extends Expression with ExpectsInputTypes { override def nullable: Boolean = true override def toString: String = s"UDF(${children.mkString(",")})" - override def inputTypes: Seq[DataType] = expectedInputTypes - // scalastyle:off /** This method has been generated by this script diff --git a/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala b/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala index e916e0776fe04..d35d37d017198 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala @@ -87,7 +87,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { (0 to 22).map { x => val types = (1 to x).foldRight("RT")((i, s) => {s"A$i, $s"}) val typeTags = (1 to x).map(i => s"A${i}: TypeTag").foldLeft("RT: TypeTag")(_ + ", " + _) - val inputTypes = (1 to x).foldLeft("Nil")((s, i) => {s"$s :+ ScalaReflection.schemaFor[A$i].dataType"}) + val inputTypes = (1 to x).foldRight("Nil")((i, s) => {s"ScalaReflection.schemaFor[A$i].dataType :: $s"}) println(s""" /** * Register a Scala closure of ${x} arguments as user-defined function (UDF). @@ -141,7 +141,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag](name: String, func: Function1[A1, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType :: Nil).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -154,7 +154,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag](name: String, func: Function2[A1, A2, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType :: ScalaReflection.schemaFor[A2].dataType :: Nil).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -167,7 +167,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag](name: String, func: Function3[A1, A2, A3, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType :: ScalaReflection.schemaFor[A2].dataType :: ScalaReflection.schemaFor[A3].dataType :: Nil).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -180,7 +180,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag](name: String, func: Function4[A1, A2, A3, A4, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType :: ScalaReflection.schemaFor[A2].dataType :: ScalaReflection.schemaFor[A3].dataType :: ScalaReflection.schemaFor[A4].dataType :: Nil).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -193,7 +193,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag](name: String, func: Function5[A1, A2, A3, A4, A5, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType :: ScalaReflection.schemaFor[A2].dataType :: ScalaReflection.schemaFor[A3].dataType :: ScalaReflection.schemaFor[A4].dataType :: ScalaReflection.schemaFor[A5].dataType :: Nil).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -206,7 +206,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag](name: String, func: Function6[A1, A2, A3, A4, A5, A6, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType :: ScalaReflection.schemaFor[A2].dataType :: ScalaReflection.schemaFor[A3].dataType :: ScalaReflection.schemaFor[A4].dataType :: ScalaReflection.schemaFor[A5].dataType :: ScalaReflection.schemaFor[A6].dataType :: Nil).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -219,7 +219,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag](name: String, func: Function7[A1, A2, A3, A4, A5, A6, A7, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType :: ScalaReflection.schemaFor[A2].dataType :: ScalaReflection.schemaFor[A3].dataType :: ScalaReflection.schemaFor[A4].dataType :: ScalaReflection.schemaFor[A5].dataType :: ScalaReflection.schemaFor[A6].dataType :: ScalaReflection.schemaFor[A7].dataType :: Nil).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -232,7 +232,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag](name: String, func: Function8[A1, A2, A3, A4, A5, A6, A7, A8, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType :+ ScalaReflection.schemaFor[A8].dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType :: ScalaReflection.schemaFor[A2].dataType :: ScalaReflection.schemaFor[A3].dataType :: ScalaReflection.schemaFor[A4].dataType :: ScalaReflection.schemaFor[A5].dataType :: ScalaReflection.schemaFor[A6].dataType :: ScalaReflection.schemaFor[A7].dataType :: ScalaReflection.schemaFor[A8].dataType :: Nil).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -245,7 +245,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag](name: String, func: Function9[A1, A2, A3, A4, A5, A6, A7, A8, A9, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType :+ ScalaReflection.schemaFor[A8].dataType :+ ScalaReflection.schemaFor[A9].dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType :: ScalaReflection.schemaFor[A2].dataType :: ScalaReflection.schemaFor[A3].dataType :: ScalaReflection.schemaFor[A4].dataType :: ScalaReflection.schemaFor[A5].dataType :: ScalaReflection.schemaFor[A6].dataType :: ScalaReflection.schemaFor[A7].dataType :: ScalaReflection.schemaFor[A8].dataType :: ScalaReflection.schemaFor[A9].dataType :: Nil).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -258,7 +258,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag](name: String, func: Function10[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType :+ ScalaReflection.schemaFor[A8].dataType :+ ScalaReflection.schemaFor[A9].dataType :+ ScalaReflection.schemaFor[A10].dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType :: ScalaReflection.schemaFor[A2].dataType :: ScalaReflection.schemaFor[A3].dataType :: ScalaReflection.schemaFor[A4].dataType :: ScalaReflection.schemaFor[A5].dataType :: ScalaReflection.schemaFor[A6].dataType :: ScalaReflection.schemaFor[A7].dataType :: ScalaReflection.schemaFor[A8].dataType :: ScalaReflection.schemaFor[A9].dataType :: ScalaReflection.schemaFor[A10].dataType :: Nil).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -271,7 +271,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag](name: String, func: Function11[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType :+ ScalaReflection.schemaFor[A8].dataType :+ ScalaReflection.schemaFor[A9].dataType :+ ScalaReflection.schemaFor[A10].dataType :+ ScalaReflection.schemaFor[A11].dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType :: ScalaReflection.schemaFor[A2].dataType :: ScalaReflection.schemaFor[A3].dataType :: ScalaReflection.schemaFor[A4].dataType :: ScalaReflection.schemaFor[A5].dataType :: ScalaReflection.schemaFor[A6].dataType :: ScalaReflection.schemaFor[A7].dataType :: ScalaReflection.schemaFor[A8].dataType :: ScalaReflection.schemaFor[A9].dataType :: ScalaReflection.schemaFor[A10].dataType :: ScalaReflection.schemaFor[A11].dataType :: Nil).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -284,7 +284,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag](name: String, func: Function12[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType :+ ScalaReflection.schemaFor[A8].dataType :+ ScalaReflection.schemaFor[A9].dataType :+ ScalaReflection.schemaFor[A10].dataType :+ ScalaReflection.schemaFor[A11].dataType :+ ScalaReflection.schemaFor[A12].dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType :: ScalaReflection.schemaFor[A2].dataType :: ScalaReflection.schemaFor[A3].dataType :: ScalaReflection.schemaFor[A4].dataType :: ScalaReflection.schemaFor[A5].dataType :: ScalaReflection.schemaFor[A6].dataType :: ScalaReflection.schemaFor[A7].dataType :: ScalaReflection.schemaFor[A8].dataType :: ScalaReflection.schemaFor[A9].dataType :: ScalaReflection.schemaFor[A10].dataType :: ScalaReflection.schemaFor[A11].dataType :: ScalaReflection.schemaFor[A12].dataType :: Nil).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -297,7 +297,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag](name: String, func: Function13[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType :+ ScalaReflection.schemaFor[A8].dataType :+ ScalaReflection.schemaFor[A9].dataType :+ ScalaReflection.schemaFor[A10].dataType :+ ScalaReflection.schemaFor[A11].dataType :+ ScalaReflection.schemaFor[A12].dataType :+ ScalaReflection.schemaFor[A13].dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType :: ScalaReflection.schemaFor[A2].dataType :: ScalaReflection.schemaFor[A3].dataType :: ScalaReflection.schemaFor[A4].dataType :: ScalaReflection.schemaFor[A5].dataType :: ScalaReflection.schemaFor[A6].dataType :: ScalaReflection.schemaFor[A7].dataType :: ScalaReflection.schemaFor[A8].dataType :: ScalaReflection.schemaFor[A9].dataType :: ScalaReflection.schemaFor[A10].dataType :: ScalaReflection.schemaFor[A11].dataType :: ScalaReflection.schemaFor[A12].dataType :: ScalaReflection.schemaFor[A13].dataType :: Nil).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -310,7 +310,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag](name: String, func: Function14[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType :+ ScalaReflection.schemaFor[A8].dataType :+ ScalaReflection.schemaFor[A9].dataType :+ ScalaReflection.schemaFor[A10].dataType :+ ScalaReflection.schemaFor[A11].dataType :+ ScalaReflection.schemaFor[A12].dataType :+ ScalaReflection.schemaFor[A13].dataType :+ ScalaReflection.schemaFor[A14].dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType :: ScalaReflection.schemaFor[A2].dataType :: ScalaReflection.schemaFor[A3].dataType :: ScalaReflection.schemaFor[A4].dataType :: ScalaReflection.schemaFor[A5].dataType :: ScalaReflection.schemaFor[A6].dataType :: ScalaReflection.schemaFor[A7].dataType :: ScalaReflection.schemaFor[A8].dataType :: ScalaReflection.schemaFor[A9].dataType :: ScalaReflection.schemaFor[A10].dataType :: ScalaReflection.schemaFor[A11].dataType :: ScalaReflection.schemaFor[A12].dataType :: ScalaReflection.schemaFor[A13].dataType :: ScalaReflection.schemaFor[A14].dataType :: Nil).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -323,7 +323,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag](name: String, func: Function15[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType :+ ScalaReflection.schemaFor[A8].dataType :+ ScalaReflection.schemaFor[A9].dataType :+ ScalaReflection.schemaFor[A10].dataType :+ ScalaReflection.schemaFor[A11].dataType :+ ScalaReflection.schemaFor[A12].dataType :+ ScalaReflection.schemaFor[A13].dataType :+ ScalaReflection.schemaFor[A14].dataType :+ ScalaReflection.schemaFor[A15].dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType :: ScalaReflection.schemaFor[A2].dataType :: ScalaReflection.schemaFor[A3].dataType :: ScalaReflection.schemaFor[A4].dataType :: ScalaReflection.schemaFor[A5].dataType :: ScalaReflection.schemaFor[A6].dataType :: ScalaReflection.schemaFor[A7].dataType :: ScalaReflection.schemaFor[A8].dataType :: ScalaReflection.schemaFor[A9].dataType :: ScalaReflection.schemaFor[A10].dataType :: ScalaReflection.schemaFor[A11].dataType :: ScalaReflection.schemaFor[A12].dataType :: ScalaReflection.schemaFor[A13].dataType :: ScalaReflection.schemaFor[A14].dataType :: ScalaReflection.schemaFor[A15].dataType :: Nil).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -336,7 +336,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag](name: String, func: Function16[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType :+ ScalaReflection.schemaFor[A8].dataType :+ ScalaReflection.schemaFor[A9].dataType :+ ScalaReflection.schemaFor[A10].dataType :+ ScalaReflection.schemaFor[A11].dataType :+ ScalaReflection.schemaFor[A12].dataType :+ ScalaReflection.schemaFor[A13].dataType :+ ScalaReflection.schemaFor[A14].dataType :+ ScalaReflection.schemaFor[A15].dataType :+ ScalaReflection.schemaFor[A16].dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType :: ScalaReflection.schemaFor[A2].dataType :: ScalaReflection.schemaFor[A3].dataType :: ScalaReflection.schemaFor[A4].dataType :: ScalaReflection.schemaFor[A5].dataType :: ScalaReflection.schemaFor[A6].dataType :: ScalaReflection.schemaFor[A7].dataType :: ScalaReflection.schemaFor[A8].dataType :: ScalaReflection.schemaFor[A9].dataType :: ScalaReflection.schemaFor[A10].dataType :: ScalaReflection.schemaFor[A11].dataType :: ScalaReflection.schemaFor[A12].dataType :: ScalaReflection.schemaFor[A13].dataType :: ScalaReflection.schemaFor[A14].dataType :: ScalaReflection.schemaFor[A15].dataType :: ScalaReflection.schemaFor[A16].dataType :: Nil).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -349,7 +349,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag](name: String, func: Function17[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType :+ ScalaReflection.schemaFor[A8].dataType :+ ScalaReflection.schemaFor[A9].dataType :+ ScalaReflection.schemaFor[A10].dataType :+ ScalaReflection.schemaFor[A11].dataType :+ ScalaReflection.schemaFor[A12].dataType :+ ScalaReflection.schemaFor[A13].dataType :+ ScalaReflection.schemaFor[A14].dataType :+ ScalaReflection.schemaFor[A15].dataType :+ ScalaReflection.schemaFor[A16].dataType :+ ScalaReflection.schemaFor[A17].dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType :: ScalaReflection.schemaFor[A2].dataType :: ScalaReflection.schemaFor[A3].dataType :: ScalaReflection.schemaFor[A4].dataType :: ScalaReflection.schemaFor[A5].dataType :: ScalaReflection.schemaFor[A6].dataType :: ScalaReflection.schemaFor[A7].dataType :: ScalaReflection.schemaFor[A8].dataType :: ScalaReflection.schemaFor[A9].dataType :: ScalaReflection.schemaFor[A10].dataType :: ScalaReflection.schemaFor[A11].dataType :: ScalaReflection.schemaFor[A12].dataType :: ScalaReflection.schemaFor[A13].dataType :: ScalaReflection.schemaFor[A14].dataType :: ScalaReflection.schemaFor[A15].dataType :: ScalaReflection.schemaFor[A16].dataType :: ScalaReflection.schemaFor[A17].dataType :: Nil).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -362,7 +362,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag](name: String, func: Function18[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType :+ ScalaReflection.schemaFor[A8].dataType :+ ScalaReflection.schemaFor[A9].dataType :+ ScalaReflection.schemaFor[A10].dataType :+ ScalaReflection.schemaFor[A11].dataType :+ ScalaReflection.schemaFor[A12].dataType :+ ScalaReflection.schemaFor[A13].dataType :+ ScalaReflection.schemaFor[A14].dataType :+ ScalaReflection.schemaFor[A15].dataType :+ ScalaReflection.schemaFor[A16].dataType :+ ScalaReflection.schemaFor[A17].dataType :+ ScalaReflection.schemaFor[A18].dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType :: ScalaReflection.schemaFor[A2].dataType :: ScalaReflection.schemaFor[A3].dataType :: ScalaReflection.schemaFor[A4].dataType :: ScalaReflection.schemaFor[A5].dataType :: ScalaReflection.schemaFor[A6].dataType :: ScalaReflection.schemaFor[A7].dataType :: ScalaReflection.schemaFor[A8].dataType :: ScalaReflection.schemaFor[A9].dataType :: ScalaReflection.schemaFor[A10].dataType :: ScalaReflection.schemaFor[A11].dataType :: ScalaReflection.schemaFor[A12].dataType :: ScalaReflection.schemaFor[A13].dataType :: ScalaReflection.schemaFor[A14].dataType :: ScalaReflection.schemaFor[A15].dataType :: ScalaReflection.schemaFor[A16].dataType :: ScalaReflection.schemaFor[A17].dataType :: ScalaReflection.schemaFor[A18].dataType :: Nil).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -375,7 +375,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag](name: String, func: Function19[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType :+ ScalaReflection.schemaFor[A8].dataType :+ ScalaReflection.schemaFor[A9].dataType :+ ScalaReflection.schemaFor[A10].dataType :+ ScalaReflection.schemaFor[A11].dataType :+ ScalaReflection.schemaFor[A12].dataType :+ ScalaReflection.schemaFor[A13].dataType :+ ScalaReflection.schemaFor[A14].dataType :+ ScalaReflection.schemaFor[A15].dataType :+ ScalaReflection.schemaFor[A16].dataType :+ ScalaReflection.schemaFor[A17].dataType :+ ScalaReflection.schemaFor[A18].dataType :+ ScalaReflection.schemaFor[A19].dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType :: ScalaReflection.schemaFor[A2].dataType :: ScalaReflection.schemaFor[A3].dataType :: ScalaReflection.schemaFor[A4].dataType :: ScalaReflection.schemaFor[A5].dataType :: ScalaReflection.schemaFor[A6].dataType :: ScalaReflection.schemaFor[A7].dataType :: ScalaReflection.schemaFor[A8].dataType :: ScalaReflection.schemaFor[A9].dataType :: ScalaReflection.schemaFor[A10].dataType :: ScalaReflection.schemaFor[A11].dataType :: ScalaReflection.schemaFor[A12].dataType :: ScalaReflection.schemaFor[A13].dataType :: ScalaReflection.schemaFor[A14].dataType :: ScalaReflection.schemaFor[A15].dataType :: ScalaReflection.schemaFor[A16].dataType :: ScalaReflection.schemaFor[A17].dataType :: ScalaReflection.schemaFor[A18].dataType :: ScalaReflection.schemaFor[A19].dataType :: Nil).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -388,7 +388,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag](name: String, func: Function20[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType :+ ScalaReflection.schemaFor[A8].dataType :+ ScalaReflection.schemaFor[A9].dataType :+ ScalaReflection.schemaFor[A10].dataType :+ ScalaReflection.schemaFor[A11].dataType :+ ScalaReflection.schemaFor[A12].dataType :+ ScalaReflection.schemaFor[A13].dataType :+ ScalaReflection.schemaFor[A14].dataType :+ ScalaReflection.schemaFor[A15].dataType :+ ScalaReflection.schemaFor[A16].dataType :+ ScalaReflection.schemaFor[A17].dataType :+ ScalaReflection.schemaFor[A18].dataType :+ ScalaReflection.schemaFor[A19].dataType :+ ScalaReflection.schemaFor[A20].dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType :: ScalaReflection.schemaFor[A2].dataType :: ScalaReflection.schemaFor[A3].dataType :: ScalaReflection.schemaFor[A4].dataType :: ScalaReflection.schemaFor[A5].dataType :: ScalaReflection.schemaFor[A6].dataType :: ScalaReflection.schemaFor[A7].dataType :: ScalaReflection.schemaFor[A8].dataType :: ScalaReflection.schemaFor[A9].dataType :: ScalaReflection.schemaFor[A10].dataType :: ScalaReflection.schemaFor[A11].dataType :: ScalaReflection.schemaFor[A12].dataType :: ScalaReflection.schemaFor[A13].dataType :: ScalaReflection.schemaFor[A14].dataType :: ScalaReflection.schemaFor[A15].dataType :: ScalaReflection.schemaFor[A16].dataType :: ScalaReflection.schemaFor[A17].dataType :: ScalaReflection.schemaFor[A18].dataType :: ScalaReflection.schemaFor[A19].dataType :: ScalaReflection.schemaFor[A20].dataType :: Nil).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -401,7 +401,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag, A21: TypeTag](name: String, func: Function21[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, A21, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType :+ ScalaReflection.schemaFor[A8].dataType :+ ScalaReflection.schemaFor[A9].dataType :+ ScalaReflection.schemaFor[A10].dataType :+ ScalaReflection.schemaFor[A11].dataType :+ ScalaReflection.schemaFor[A12].dataType :+ ScalaReflection.schemaFor[A13].dataType :+ ScalaReflection.schemaFor[A14].dataType :+ ScalaReflection.schemaFor[A15].dataType :+ ScalaReflection.schemaFor[A16].dataType :+ ScalaReflection.schemaFor[A17].dataType :+ ScalaReflection.schemaFor[A18].dataType :+ ScalaReflection.schemaFor[A19].dataType :+ ScalaReflection.schemaFor[A20].dataType :+ ScalaReflection.schemaFor[A21].dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType :: ScalaReflection.schemaFor[A2].dataType :: ScalaReflection.schemaFor[A3].dataType :: ScalaReflection.schemaFor[A4].dataType :: ScalaReflection.schemaFor[A5].dataType :: ScalaReflection.schemaFor[A6].dataType :: ScalaReflection.schemaFor[A7].dataType :: ScalaReflection.schemaFor[A8].dataType :: ScalaReflection.schemaFor[A9].dataType :: ScalaReflection.schemaFor[A10].dataType :: ScalaReflection.schemaFor[A11].dataType :: ScalaReflection.schemaFor[A12].dataType :: ScalaReflection.schemaFor[A13].dataType :: ScalaReflection.schemaFor[A14].dataType :: ScalaReflection.schemaFor[A15].dataType :: ScalaReflection.schemaFor[A16].dataType :: ScalaReflection.schemaFor[A17].dataType :: ScalaReflection.schemaFor[A18].dataType :: ScalaReflection.schemaFor[A19].dataType :: ScalaReflection.schemaFor[A20].dataType :: ScalaReflection.schemaFor[A21].dataType :: Nil).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) @@ -414,7 +414,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging { */ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag, A21: TypeTag, A22: TypeTag](name: String, func: Function22[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, A21, A22, RT]): UserDefinedFunction = { val dataType = ScalaReflection.schemaFor[RT].dataType - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor[A1].dataType :+ ScalaReflection.schemaFor[A2].dataType :+ ScalaReflection.schemaFor[A3].dataType :+ ScalaReflection.schemaFor[A4].dataType :+ ScalaReflection.schemaFor[A5].dataType :+ ScalaReflection.schemaFor[A6].dataType :+ ScalaReflection.schemaFor[A7].dataType :+ ScalaReflection.schemaFor[A8].dataType :+ ScalaReflection.schemaFor[A9].dataType :+ ScalaReflection.schemaFor[A10].dataType :+ ScalaReflection.schemaFor[A11].dataType :+ ScalaReflection.schemaFor[A12].dataType :+ ScalaReflection.schemaFor[A13].dataType :+ ScalaReflection.schemaFor[A14].dataType :+ ScalaReflection.schemaFor[A15].dataType :+ ScalaReflection.schemaFor[A16].dataType :+ ScalaReflection.schemaFor[A17].dataType :+ ScalaReflection.schemaFor[A18].dataType :+ ScalaReflection.schemaFor[A19].dataType :+ ScalaReflection.schemaFor[A20].dataType :+ ScalaReflection.schemaFor[A21].dataType :+ ScalaReflection.schemaFor[A22].dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType :: ScalaReflection.schemaFor[A2].dataType :: ScalaReflection.schemaFor[A3].dataType :: ScalaReflection.schemaFor[A4].dataType :: ScalaReflection.schemaFor[A5].dataType :: ScalaReflection.schemaFor[A6].dataType :: ScalaReflection.schemaFor[A7].dataType :: ScalaReflection.schemaFor[A8].dataType :: ScalaReflection.schemaFor[A9].dataType :: ScalaReflection.schemaFor[A10].dataType :: ScalaReflection.schemaFor[A11].dataType :: ScalaReflection.schemaFor[A12].dataType :: ScalaReflection.schemaFor[A13].dataType :: ScalaReflection.schemaFor[A14].dataType :: ScalaReflection.schemaFor[A15].dataType :: ScalaReflection.schemaFor[A16].dataType :: ScalaReflection.schemaFor[A17].dataType :: ScalaReflection.schemaFor[A18].dataType :: ScalaReflection.schemaFor[A19].dataType :: ScalaReflection.schemaFor[A20].dataType :: ScalaReflection.schemaFor[A21].dataType :: ScalaReflection.schemaFor[A22].dataType :: Nil).getOrElse(Nil) def builder(e: Seq[Expression]) = ScalaUDF(func, dataType, e, inputTypes) functionRegistry.registerFunction(name, builder) UserDefinedFunction(func, dataType) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala index 57f7f5ff55a8d..898434df4f402 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala @@ -1549,7 +1549,7 @@ object functions { (0 to 10).map { x => val types = (1 to x).foldRight("RT")((i, s) => {s"A$i, $s"}) val typeTags = (1 to x).map(i => s"A$i: TypeTag").foldLeft("RT: TypeTag")(_ + ", " + _) - val inputTypes = (1 to x).foldLeft("Nil")((s, i) => {s"$s :+ ScalaReflection.schemaFor(typeTag[A$i]).dataType"}) + val inputTypes = (1 to x).foldRight("Nil")((i, s) => {s"ScalaReflection.schemaFor(typeTag[A$i]).dataType :: $s"}) println(s""" /** * Defines a user-defined function of ${x} arguments as user-defined function (UDF). @@ -1604,7 +1604,7 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag, A1: TypeTag](f: Function1[A1, RT]): UserDefinedFunction = { - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor(typeTag[A1]).dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor(typeTag[A1]).dataType :: Nil).getOrElse(Nil) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } @@ -1616,7 +1616,7 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag](f: Function2[A1, A2, RT]): UserDefinedFunction = { - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor(typeTag[A1]).dataType :+ ScalaReflection.schemaFor(typeTag[A2]).dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor(typeTag[A1]).dataType :: ScalaReflection.schemaFor(typeTag[A2]).dataType :: Nil).getOrElse(Nil) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } @@ -1628,7 +1628,7 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag](f: Function3[A1, A2, A3, RT]): UserDefinedFunction = { - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor(typeTag[A1]).dataType :+ ScalaReflection.schemaFor(typeTag[A2]).dataType :+ ScalaReflection.schemaFor(typeTag[A3]).dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor(typeTag[A1]).dataType :: ScalaReflection.schemaFor(typeTag[A2]).dataType :: ScalaReflection.schemaFor(typeTag[A3]).dataType :: Nil).getOrElse(Nil) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } @@ -1640,7 +1640,7 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag](f: Function4[A1, A2, A3, A4, RT]): UserDefinedFunction = { - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor(typeTag[A1]).dataType :+ ScalaReflection.schemaFor(typeTag[A2]).dataType :+ ScalaReflection.schemaFor(typeTag[A3]).dataType :+ ScalaReflection.schemaFor(typeTag[A4]).dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor(typeTag[A1]).dataType :: ScalaReflection.schemaFor(typeTag[A2]).dataType :: ScalaReflection.schemaFor(typeTag[A3]).dataType :: ScalaReflection.schemaFor(typeTag[A4]).dataType :: Nil).getOrElse(Nil) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } @@ -1652,7 +1652,7 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag](f: Function5[A1, A2, A3, A4, A5, RT]): UserDefinedFunction = { - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor(typeTag[A1]).dataType :+ ScalaReflection.schemaFor(typeTag[A2]).dataType :+ ScalaReflection.schemaFor(typeTag[A3]).dataType :+ ScalaReflection.schemaFor(typeTag[A4]).dataType :+ ScalaReflection.schemaFor(typeTag[A5]).dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor(typeTag[A1]).dataType :: ScalaReflection.schemaFor(typeTag[A2]).dataType :: ScalaReflection.schemaFor(typeTag[A3]).dataType :: ScalaReflection.schemaFor(typeTag[A4]).dataType :: ScalaReflection.schemaFor(typeTag[A5]).dataType :: Nil).getOrElse(Nil) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } @@ -1664,7 +1664,7 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag](f: Function6[A1, A2, A3, A4, A5, A6, RT]): UserDefinedFunction = { - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor(typeTag[A1]).dataType :+ ScalaReflection.schemaFor(typeTag[A2]).dataType :+ ScalaReflection.schemaFor(typeTag[A3]).dataType :+ ScalaReflection.schemaFor(typeTag[A4]).dataType :+ ScalaReflection.schemaFor(typeTag[A5]).dataType :+ ScalaReflection.schemaFor(typeTag[A6]).dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor(typeTag[A1]).dataType :: ScalaReflection.schemaFor(typeTag[A2]).dataType :: ScalaReflection.schemaFor(typeTag[A3]).dataType :: ScalaReflection.schemaFor(typeTag[A4]).dataType :: ScalaReflection.schemaFor(typeTag[A5]).dataType :: ScalaReflection.schemaFor(typeTag[A6]).dataType :: Nil).getOrElse(Nil) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } @@ -1676,7 +1676,7 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag](f: Function7[A1, A2, A3, A4, A5, A6, A7, RT]): UserDefinedFunction = { - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor(typeTag[A1]).dataType :+ ScalaReflection.schemaFor(typeTag[A2]).dataType :+ ScalaReflection.schemaFor(typeTag[A3]).dataType :+ ScalaReflection.schemaFor(typeTag[A4]).dataType :+ ScalaReflection.schemaFor(typeTag[A5]).dataType :+ ScalaReflection.schemaFor(typeTag[A6]).dataType :+ ScalaReflection.schemaFor(typeTag[A7]).dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor(typeTag[A1]).dataType :: ScalaReflection.schemaFor(typeTag[A2]).dataType :: ScalaReflection.schemaFor(typeTag[A3]).dataType :: ScalaReflection.schemaFor(typeTag[A4]).dataType :: ScalaReflection.schemaFor(typeTag[A5]).dataType :: ScalaReflection.schemaFor(typeTag[A6]).dataType :: ScalaReflection.schemaFor(typeTag[A7]).dataType :: Nil).getOrElse(Nil) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } @@ -1688,7 +1688,7 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag](f: Function8[A1, A2, A3, A4, A5, A6, A7, A8, RT]): UserDefinedFunction = { - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor(typeTag[A1]).dataType :+ ScalaReflection.schemaFor(typeTag[A2]).dataType :+ ScalaReflection.schemaFor(typeTag[A3]).dataType :+ ScalaReflection.schemaFor(typeTag[A4]).dataType :+ ScalaReflection.schemaFor(typeTag[A5]).dataType :+ ScalaReflection.schemaFor(typeTag[A6]).dataType :+ ScalaReflection.schemaFor(typeTag[A7]).dataType :+ ScalaReflection.schemaFor(typeTag[A8]).dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor(typeTag[A1]).dataType :: ScalaReflection.schemaFor(typeTag[A2]).dataType :: ScalaReflection.schemaFor(typeTag[A3]).dataType :: ScalaReflection.schemaFor(typeTag[A4]).dataType :: ScalaReflection.schemaFor(typeTag[A5]).dataType :: ScalaReflection.schemaFor(typeTag[A6]).dataType :: ScalaReflection.schemaFor(typeTag[A7]).dataType :: ScalaReflection.schemaFor(typeTag[A8]).dataType :: Nil).getOrElse(Nil) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } @@ -1700,7 +1700,7 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag](f: Function9[A1, A2, A3, A4, A5, A6, A7, A8, A9, RT]): UserDefinedFunction = { - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor(typeTag[A1]).dataType :+ ScalaReflection.schemaFor(typeTag[A2]).dataType :+ ScalaReflection.schemaFor(typeTag[A3]).dataType :+ ScalaReflection.schemaFor(typeTag[A4]).dataType :+ ScalaReflection.schemaFor(typeTag[A5]).dataType :+ ScalaReflection.schemaFor(typeTag[A6]).dataType :+ ScalaReflection.schemaFor(typeTag[A7]).dataType :+ ScalaReflection.schemaFor(typeTag[A8]).dataType :+ ScalaReflection.schemaFor(typeTag[A9]).dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor(typeTag[A1]).dataType :: ScalaReflection.schemaFor(typeTag[A2]).dataType :: ScalaReflection.schemaFor(typeTag[A3]).dataType :: ScalaReflection.schemaFor(typeTag[A4]).dataType :: ScalaReflection.schemaFor(typeTag[A5]).dataType :: ScalaReflection.schemaFor(typeTag[A6]).dataType :: ScalaReflection.schemaFor(typeTag[A7]).dataType :: ScalaReflection.schemaFor(typeTag[A8]).dataType :: ScalaReflection.schemaFor(typeTag[A9]).dataType :: Nil).getOrElse(Nil) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) } @@ -1712,7 +1712,7 @@ object functions { * @since 1.3.0 */ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag](f: Function10[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, RT]): UserDefinedFunction = { - val inputTypes = Try(Nil :+ ScalaReflection.schemaFor(typeTag[A1]).dataType :+ ScalaReflection.schemaFor(typeTag[A2]).dataType :+ ScalaReflection.schemaFor(typeTag[A3]).dataType :+ ScalaReflection.schemaFor(typeTag[A4]).dataType :+ ScalaReflection.schemaFor(typeTag[A5]).dataType :+ ScalaReflection.schemaFor(typeTag[A6]).dataType :+ ScalaReflection.schemaFor(typeTag[A7]).dataType :+ ScalaReflection.schemaFor(typeTag[A8]).dataType :+ ScalaReflection.schemaFor(typeTag[A9]).dataType :+ ScalaReflection.schemaFor(typeTag[A10]).dataType).getOrElse(Nil) + val inputTypes = Try(ScalaReflection.schemaFor(typeTag[A1]).dataType :: ScalaReflection.schemaFor(typeTag[A2]).dataType :: ScalaReflection.schemaFor(typeTag[A3]).dataType :: ScalaReflection.schemaFor(typeTag[A4]).dataType :: ScalaReflection.schemaFor(typeTag[A5]).dataType :: ScalaReflection.schemaFor(typeTag[A6]).dataType :: ScalaReflection.schemaFor(typeTag[A7]).dataType :: ScalaReflection.schemaFor(typeTag[A8]).dataType :: ScalaReflection.schemaFor(typeTag[A9]).dataType :: ScalaReflection.schemaFor(typeTag[A10]).dataType :: Nil).getOrElse(Nil) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, inputTypes) }