From 101f3451e07d64cb650699e3b49b1c35fc7cce20 Mon Sep 17 00:00:00 2001
From: Efim Poberezkin <efim@poberezkin.ru>
Date: Wed, 14 Mar 2018 14:22:59 +0400
Subject: [PATCH 1/2] [SPARK-20536][SQL] Extend ColumnName to create
 StructFields with explicit nullable

---
 .../scala/org/apache/spark/sql/Column.scala   | 100 ++++++++++++++++++
 1 file changed, 100 insertions(+)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
index 92988680871a4..88045ddc8552f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
@@ -1208,60 +1208,121 @@ class ColumnName(name: String) extends Column(name) {
    */
   def boolean: StructField = StructField(name, BooleanType)
 
+  /**
+   * Creates a new `StructField` of type boolean.
+   * @since 2.3.0
+   */
+  def boolean(nullable: Boolean): StructField = StructField(name, BooleanType, nullable)
+
   /**
    * Creates a new `StructField` of type byte.
    * @since 1.3.0
    */
   def byte: StructField = StructField(name, ByteType)
 
+  /**
+   * Creates a new `StructField` of type byte.
+   * @since 2.3.0
+   */
+  def byte(nullable: Boolean): StructField = StructField(name, ByteType, nullable)
+
   /**
    * Creates a new `StructField` of type short.
    * @since 1.3.0
    */
   def short: StructField = StructField(name, ShortType)
 
+  /**
+   * Creates a new `StructField` of type short.
+   * @since 2.3.0
+   */
+  def short(nullable: Boolean): StructField = StructField(name, ShortType, nullable)
+
   /**
    * Creates a new `StructField` of type int.
    * @since 1.3.0
    */
   def int: StructField = StructField(name, IntegerType)
 
+  /**
+   * Creates a new `StructField` of type int.
+   * @since 2.3.0
+   */
+  def int(nullable: Boolean): StructField = StructField(name, IntegerType, nullable)
+
   /**
    * Creates a new `StructField` of type long.
    * @since 1.3.0
    */
   def long: StructField = StructField(name, LongType)
 
+  /**
+   * Creates a new `StructField` of type long.
+   * @since 2.3.0
+   */
+  def long(nullable: Boolean): StructField = StructField(name, LongType, nullable)
+
   /**
    * Creates a new `StructField` of type float.
    * @since 1.3.0
    */
   def float: StructField = StructField(name, FloatType)
 
+  /**
+   * Creates a new `StructField` of type float.
+   * @since 2.3.0
+   */
+  def float(nullable: Boolean): StructField = StructField(name, FloatType, nullable)
+
   /**
    * Creates a new `StructField` of type double.
    * @since 1.3.0
    */
   def double: StructField = StructField(name, DoubleType)
 
+  /**
+   * Creates a new `StructField` of type double.
+   * @since 2.3.0
+   */
+  def double(nullable: Boolean): StructField = StructField(name, DoubleType, nullable)
+
   /**
    * Creates a new `StructField` of type string.
    * @since 1.3.0
    */
   def string: StructField = StructField(name, StringType)
 
+  /**
+   * Creates a new `StructField` of type string.
+   * @since 2.3.0
+   */
+  def string(nullable: Boolean): StructField = StructField(name, StringType, nullable)
+
   /**
    * Creates a new `StructField` of type date.
    * @since 1.3.0
    */
   def date: StructField = StructField(name, DateType)
 
+  /**
+   * Creates a new `StructField` of type date.
+   * @since 2.3.0
+   */
+  def date(nullable: Boolean): StructField = StructField(name, DateType, nullable)
+
   /**
    * Creates a new `StructField` of type decimal.
    * @since 1.3.0
    */
   def decimal: StructField = StructField(name, DecimalType.USER_DEFAULT)
 
+  /**
+   * Creates a new `StructField` of type decimal.
+   * @since 2.3.0
+   */
+  def decimal(nullable: Boolean): StructField =
+    StructField(name, DecimalType.USER_DEFAULT, nullable)
+
   /**
    * Creates a new `StructField` of type decimal.
    * @since 1.3.0
@@ -1269,24 +1330,50 @@ class ColumnName(name: String) extends Column(name) {
   def decimal(precision: Int, scale: Int): StructField =
     StructField(name, DecimalType(precision, scale))
 
+  /**
+   * Creates a new `StructField` of type decimal.
+   * @since 2.3.0
+   */
+  def decimal(precision: Int, scale: Int, nullable: Boolean): StructField =
+    StructField(name, DecimalType(precision, scale), nullable)
+
   /**
    * Creates a new `StructField` of type timestamp.
    * @since 1.3.0
    */
   def timestamp: StructField = StructField(name, TimestampType)
 
+  /**
+   * Creates a new `StructField` of type timestamp.
+   * @since 2.3.0
+   */
+  def timestamp(nullable: Boolean): StructField = StructField(name, TimestampType, nullable)
+
   /**
    * Creates a new `StructField` of type binary.
    * @since 1.3.0
    */
   def binary: StructField = StructField(name, BinaryType)
 
+  /**
+   * Creates a new `StructField` of type binary.
+   * @since 2.3.0
+   */
+  def binary(nullable: Boolean): StructField = StructField(name, BinaryType, nullable)
+
   /**
    * Creates a new `StructField` of type array.
    * @since 1.3.0
    */
   def array(dataType: DataType): StructField = StructField(name, ArrayType(dataType))
 
+  /**
+   * Creates a new `StructField` of type array.
+   * @since 2.3.0
+   */
+  def array(dataType: DataType, nullable: Boolean): StructField =
+    StructField(name, ArrayType(dataType), nullable)
+
   /**
    * Creates a new `StructField` of type map.
    * @since 1.3.0
@@ -1296,6 +1383,12 @@ class ColumnName(name: String) extends Column(name) {
 
   def map(mapType: MapType): StructField = StructField(name, mapType)
 
+  /**
+   * Creates a new `StructField` of type map.
+   * @since 2.3.0
+   */
+  def map(mapType: MapType, nullable: Boolean): StructField = StructField(name, mapType, nullable)
+
   /**
    * Creates a new `StructField` of type struct.
    * @since 1.3.0
@@ -1307,4 +1400,11 @@ class ColumnName(name: String) extends Column(name) {
    * @since 1.3.0
    */
   def struct(structType: StructType): StructField = StructField(name, structType)
+
+  /**
+   * Creates a new `StructField` of type struct.
+   * @since 2.3.0
+   */
+  def struct(structType: StructType, nullable: Boolean): StructField =
+    StructField(name, structType, nullable)
 }

From d409e95485fbcee2fe09399310ece37caa37e5af Mon Sep 17 00:00:00 2001
From: Efim Poberezkin <efim@poberezkin.ru>
Date: Thu, 15 Mar 2018 12:25:49 +0400
Subject: [PATCH 2/2] [SPARK-20536][SQL] Changed version to 2.4.0

---
 .../scala/org/apache/spark/sql/Column.scala   | 32 +++++++++----------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
index 88045ddc8552f..ed5e93712282b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
@@ -1210,7 +1210,7 @@ class ColumnName(name: String) extends Column(name) {
 
   /**
    * Creates a new `StructField` of type boolean.
-   * @since 2.3.0
+   * @since 2.4.0
    */
   def boolean(nullable: Boolean): StructField = StructField(name, BooleanType, nullable)
 
@@ -1222,7 +1222,7 @@ class ColumnName(name: String) extends Column(name) {
 
   /**
    * Creates a new `StructField` of type byte.
-   * @since 2.3.0
+   * @since 2.4.0
    */
   def byte(nullable: Boolean): StructField = StructField(name, ByteType, nullable)
 
@@ -1234,7 +1234,7 @@ class ColumnName(name: String) extends Column(name) {
 
   /**
    * Creates a new `StructField` of type short.
-   * @since 2.3.0
+   * @since 2.4.0
    */
   def short(nullable: Boolean): StructField = StructField(name, ShortType, nullable)
 
@@ -1246,7 +1246,7 @@ class ColumnName(name: String) extends Column(name) {
 
   /**
    * Creates a new `StructField` of type int.
-   * @since 2.3.0
+   * @since 2.4.0
    */
   def int(nullable: Boolean): StructField = StructField(name, IntegerType, nullable)
 
@@ -1258,7 +1258,7 @@ class ColumnName(name: String) extends Column(name) {
 
   /**
    * Creates a new `StructField` of type long.
-   * @since 2.3.0
+   * @since 2.4.0
    */
   def long(nullable: Boolean): StructField = StructField(name, LongType, nullable)
 
@@ -1270,7 +1270,7 @@ class ColumnName(name: String) extends Column(name) {
 
   /**
    * Creates a new `StructField` of type float.
-   * @since 2.3.0
+   * @since 2.4.0
    */
   def float(nullable: Boolean): StructField = StructField(name, FloatType, nullable)
 
@@ -1282,7 +1282,7 @@ class ColumnName(name: String) extends Column(name) {
 
   /**
    * Creates a new `StructField` of type double.
-   * @since 2.3.0
+   * @since 2.4.0
    */
   def double(nullable: Boolean): StructField = StructField(name, DoubleType, nullable)
 
@@ -1294,7 +1294,7 @@ class ColumnName(name: String) extends Column(name) {
 
   /**
    * Creates a new `StructField` of type string.
-   * @since 2.3.0
+   * @since 2.4.0
    */
   def string(nullable: Boolean): StructField = StructField(name, StringType, nullable)
 
@@ -1306,7 +1306,7 @@ class ColumnName(name: String) extends Column(name) {
 
   /**
    * Creates a new `StructField` of type date.
-   * @since 2.3.0
+   * @since 2.4.0
    */
   def date(nullable: Boolean): StructField = StructField(name, DateType, nullable)
 
@@ -1318,7 +1318,7 @@ class ColumnName(name: String) extends Column(name) {
 
   /**
    * Creates a new `StructField` of type decimal.
-   * @since 2.3.0
+   * @since 2.4.0
    */
   def decimal(nullable: Boolean): StructField =
     StructField(name, DecimalType.USER_DEFAULT, nullable)
@@ -1332,7 +1332,7 @@ class ColumnName(name: String) extends Column(name) {
 
   /**
    * Creates a new `StructField` of type decimal.
-   * @since 2.3.0
+   * @since 2.4.0
    */
   def decimal(precision: Int, scale: Int, nullable: Boolean): StructField =
     StructField(name, DecimalType(precision, scale), nullable)
@@ -1345,7 +1345,7 @@ class ColumnName(name: String) extends Column(name) {
 
   /**
    * Creates a new `StructField` of type timestamp.
-   * @since 2.3.0
+   * @since 2.4.0
    */
   def timestamp(nullable: Boolean): StructField = StructField(name, TimestampType, nullable)
 
@@ -1357,7 +1357,7 @@ class ColumnName(name: String) extends Column(name) {
 
   /**
    * Creates a new `StructField` of type binary.
-   * @since 2.3.0
+   * @since 2.4.0
    */
   def binary(nullable: Boolean): StructField = StructField(name, BinaryType, nullable)
 
@@ -1369,7 +1369,7 @@ class ColumnName(name: String) extends Column(name) {
 
   /**
    * Creates a new `StructField` of type array.
-   * @since 2.3.0
+   * @since 2.4.0
    */
   def array(dataType: DataType, nullable: Boolean): StructField =
     StructField(name, ArrayType(dataType), nullable)
@@ -1385,7 +1385,7 @@ class ColumnName(name: String) extends Column(name) {
 
   /**
    * Creates a new `StructField` of type map.
-   * @since 2.3.0
+   * @since 2.4.0
    */
   def map(mapType: MapType, nullable: Boolean): StructField = StructField(name, mapType, nullable)
 
@@ -1403,7 +1403,7 @@ class ColumnName(name: String) extends Column(name) {
 
   /**
    * Creates a new `StructField` of type struct.
-   * @since 2.3.0
+   * @since 2.4.0
    */
   def struct(structType: StructType, nullable: Boolean): StructField =
     StructField(name, structType, nullable)