From de207321717c607eccb280aa0b77b3393ef4a04b Mon Sep 17 00:00:00 2001 From: aditkumar Date: Fri, 9 Mar 2018 11:08:38 -0500 Subject: [PATCH 01/32] Date direction in months_between pyspark --- python/pyspark/sql/functions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py index dff590983b4d9..829572a13a4cb 100644 --- a/python/pyspark/sql/functions.py +++ b/python/pyspark/sql/functions.py @@ -1078,7 +1078,8 @@ def add_months(start, months): @since(1.5) def months_between(date1, date2): """ - Returns the number of months between date1 and date2. + Returns the number of months between date1 and date2. If date1 is later than date2, + then the result is positive. >>> df = spark.createDataFrame([('1997-02-28 10:30:00', '1996-10-30')], ['date1', 'date2']) >>> df.select(months_between(df.date1, df.date2).alias('months')).collect() From 5f8a23fea197d60f870177ed3272154eab60bca9 Mon Sep 17 00:00:00 2001 From: aditkumar Date: Fri, 9 Mar 2018 11:11:15 -0500 Subject: [PATCH 02/32] datediff direction in months_between --- R/pkg/R/functions.R | 1 + 1 file changed, 1 insertion(+) diff --git a/R/pkg/R/functions.R b/R/pkg/R/functions.R index a527426b19674..4f1a7c11fa702 100644 --- a/R/pkg/R/functions.R +++ b/R/pkg/R/functions.R @@ -1898,6 +1898,7 @@ setMethod("atan2", signature(y = "Column"), #' @details #' \code{datediff}: Returns the number of days from \code{y} to \code{x}. +#' If \code{y} is later than \code{x} then the result is positive. #' #' @rdname column_datetime_diff_functions #' @aliases datediff datediff,Column-method From 1f5c543cf8e03b32691bbe477b09c4bce02e6f4d Mon Sep 17 00:00:00 2001 From: aditkumar Date: Fri, 9 Mar 2018 11:16:01 -0500 Subject: [PATCH 03/32] date direction in months_between --- sql/core/src/main/scala/org/apache/spark/sql/functions.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala index c9ca9a8996344..e325fc75a5de5 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala @@ -2708,7 +2708,8 @@ object functions { def minute(e: Column): Column = withExpr { Minute(e.expr) } /** - * Returns number of months between dates `date1` and `date2`. + * Returns number of months between dates `date1` and `date2`. If `date1` is later than `date2`, + * then the result is positive. * @group datetime_funcs * @since 1.5.0 */ From 110f2820556eb2144f96353608d1df86ab41bc49 Mon Sep 17 00:00:00 2001 From: aditkumar Date: Thu, 15 Mar 2018 10:01:39 -0400 Subject: [PATCH 04/32] line up sql/catalyst/util/DateTimeUtils.scala --- .../src/main/scala/org/apache/spark/sql/functions.scala | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala index e325fc75a5de5..6f1e3049b5593 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala @@ -2708,8 +2708,13 @@ object functions { def minute(e: Column): Column = withExpr { Minute(e.expr) } /** - * Returns number of months between dates `date1` and `date2`. If `date1` is later than `date2`, - * then the result is positive. + * Returns number of months between dates `date1` and `date2`. + * If `date1` is later than `date2`, then the result is positive. + * If `date1` and `date2` are on the same day of month, or both are the last day of month, + * returns an integer (time of day will be ignored). + * + * Otherwise, the difference is calculated based on 31 days per month, and rounded to + * 8 digits. * @group datetime_funcs * @since 1.5.0 */ From c5b0c80897089fa4724e868ca230831569562007 Mon Sep 17 00:00:00 2001 From: aditkumar Date: Thu, 15 Mar 2018 10:06:32 -0400 Subject: [PATCH 05/32] line up with sql/catalyst/util/DateTimeUtils.scala --- .../catalyst/expressions/datetimeExpressions.scala | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 1ae4e5a2f716b..f3180c9cb55d3 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -1115,13 +1115,18 @@ case class AddMonths(startDate: Expression, numMonths: Expression) override def prettyName: String = "add_months" } - /** - * Returns number of months between dates date1 and date2. - */ + * Returns number of months between dates `date1` and `date2`. + * If `date1` is later than `date2`, then the result is positive. + * If `date1` and `date2` are on the same day of month, or both are the last day of month, + * returns an integer (time of day will be ignored). + * + * Otherwise, the difference is calculated based on 31 days per month, and rounded to + * 8 digits. +*/ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(timestamp1, timestamp2) - Returns number of months between `timestamp1` and `timestamp2`.", + usage = "_FUNC_(timestamp1, timestamp2) - Returns number of months between `timestamp1` and `timestamp2`. Positive if `timestamp1` is later than `timestamp2`", examples = """ Examples: > SELECT _FUNC_('1997-02-28 10:30:00', '1996-10-30'); From 0dc44b851ed940be243d7cc3354fabfd92c250e4 Mon Sep 17 00:00:00 2001 From: aditkumar Date: Thu, 15 Mar 2018 10:13:25 -0400 Subject: [PATCH 06/32] line up with sql/catalyst/util/DateTimeUtils.scala --- python/pyspark/sql/functions.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py index 829572a13a4cb..4e7602789fbd1 100644 --- a/python/pyspark/sql/functions.py +++ b/python/pyspark/sql/functions.py @@ -1078,8 +1078,13 @@ def add_months(start, months): @since(1.5) def months_between(date1, date2): """ - Returns the number of months between date1 and date2. If date1 is later than date2, - then the result is positive. + Returns number of months between dates date1 and date2. + If date1 is later than date2, then the result is positive. + If date1 and date2 are on the same day of month, or both are the last day of month, + returns an integer (time of day will be ignored). + + Otherwise, the difference is calculated based on 31 days per month, and rounded to + 8 digits. >>> df = spark.createDataFrame([('1997-02-28 10:30:00', '1996-10-30')], ['date1', 'date2']) >>> df.select(months_between(df.date1, df.date2).alias('months')).collect() From eadc16657bae3e59a3881640e11b14b79f1f6e90 Mon Sep 17 00:00:00 2001 From: aditkumar Date: Thu, 15 Mar 2018 10:17:17 -0400 Subject: [PATCH 07/32] cleaning up language --- .../spark/sql/catalyst/util/DateTimeUtils.scala | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala index fa69b8af62c85..929fcfb80f6e8 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala @@ -862,15 +862,15 @@ object DateTimeUtils { daysToMillis(newDays, timeZone) * 1000L - daysToMillis(days, timeZone) * 1000L + microseconds } - + /** * Returns number of months between time1 and time2. time1 and time2 are expressed in * microseconds since 1.1.1970. * - * If time1 and time2 having the same day of month, or both are the last day of month, - * it returns an integer (time under a day will be ignored). + * If time1 and time2 are on the same day of month, or both are the last day of month, + * returns an integer (time under a day will be ignored). * - * Otherwise, the difference is calculated based on 31 days per month, and rounding to + * Otherwise, the difference is calculated based on 31 days per month, and rounded to * 8 digits. */ def monthsBetween(time1: SQLTimestamp, time2: SQLTimestamp): Double = { @@ -881,10 +881,10 @@ object DateTimeUtils { * Returns number of months between time1 and time2. time1 and time2 are expressed in * microseconds since 1.1.1970. * - * If time1 and time2 having the same day of month, or both are the last day of month, - * it returns an integer (time under a day will be ignored). + * If time1 and time2 are on the same day of month, or both are the last day of month, + * returns an integer (time under a day will be ignored). * - * Otherwise, the difference is calculated based on 31 days per month, and rounding to + * Otherwise, the difference is calculated based on 31 days per month, and rounded to * 8 digits. */ def monthsBetween(time1: SQLTimestamp, time2: SQLTimestamp, timeZone: TimeZone): Double = { From 2de6acacbecdbc6a65cc956c682ae8a3ff4175d2 Mon Sep 17 00:00:00 2001 From: aditkumar Date: Thu, 15 Mar 2018 10:23:00 -0400 Subject: [PATCH 08/32] line up with sql/catalyst/util/DateTimeUtils.scala --- R/pkg/R/functions.R | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/R/pkg/R/functions.R b/R/pkg/R/functions.R index 4f1a7c11fa702..fc2cba68fd82b 100644 --- a/R/pkg/R/functions.R +++ b/R/pkg/R/functions.R @@ -1958,8 +1958,12 @@ setMethod("levenshtein", signature(y = "Column"), }) #' @details -#' \code{months_between}: Returns number of months between dates \code{y} and \code{x}. -#' +#' \code{months_between}: Returns number of months between dates \code{y} and \code{x}. +#' If \code{y} is later than \code{x}, then the result is positive. +#' If \code{y} and \code{x} are on the same day of month, or both are the last day of month, +#' returns an integer (time of day will be ignored). +#' Otherwise, the difference is calculated based on 31 days per month, and rounded to +#' 8 digits. #' @rdname column_datetime_diff_functions #' @aliases months_between months_between,Column-method #' @note months_between since 1.5.0 From 9afb3140469f801d1f08c5b30e0698ec524fcc92 Mon Sep 17 00:00:00 2001 From: aditkumar Date: Tue, 20 Mar 2018 10:44:06 -0400 Subject: [PATCH 09/32] date -> timestamp. Also removing whitespace --- .../sql/catalyst/expressions/datetimeExpressions.scala | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index f3180c9cb55d3..b0e7a4a00e7d5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -1116,13 +1116,12 @@ case class AddMonths(startDate: Expression, numMonths: Expression) override def prettyName: String = "add_months" } /** - * Returns number of months between dates `date1` and `date2`. - * If `date1` is later than `date2`, then the result is positive. - * If `date1` and `date2` are on the same day of month, or both are the last day of month, + * Returns number of months between dates `timestamp1` and `timestamp2`. + * If `timestamp` is later than `timestamp2`, then the result is positive. + * If `timestamp1` and `timestamp2` are on the same day of month, or both are the last day of month, * returns an integer (time of day will be ignored). - * * Otherwise, the difference is calculated based on 31 days per month, and rounded to - * 8 digits. + * 8 digits. */ // scalastyle:off line.size.limit @ExpressionDescription( From 23af99bc1d1b96240bf44f6ff3fe0d34be4ab402 Mon Sep 17 00:00:00 2001 From: aditkumar Date: Tue, 20 Mar 2018 10:47:32 -0400 Subject: [PATCH 10/32] remove whitespace --- .../org/apache/spark/sql/catalyst/util/DateTimeUtils.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala index 929fcfb80f6e8..0d6f9e9d9293f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala @@ -862,7 +862,7 @@ object DateTimeUtils { daysToMillis(newDays, timeZone) * 1000L - daysToMillis(days, timeZone) * 1000L + microseconds } - + /** * Returns number of months between time1 and time2. time1 and time2 are expressed in * microseconds since 1.1.1970. From 264ed994e2b74239907fdccc40544ca67d9b0531 Mon Sep 17 00:00:00 2001 From: aditkumar Date: Tue, 20 Mar 2018 10:48:54 -0400 Subject: [PATCH 11/32] remove whitespace --- sql/core/src/main/scala/org/apache/spark/sql/functions.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala index 6f1e3049b5593..1d03d86a2c465 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala @@ -2708,13 +2708,13 @@ object functions { def minute(e: Column): Column = withExpr { Minute(e.expr) } /** - * Returns number of months between dates `date1` and `date2`. + * Returns number of months between dates `date1` and `date2`. * If `date1` is later than `date2`, then the result is positive. * If `date1` and `date2` are on the same day of month, or both are the last day of month, * returns an integer (time of day will be ignored). * * Otherwise, the difference is calculated based on 31 days per month, and rounded to - * 8 digits. + * 8 digits. * @group datetime_funcs * @since 1.5.0 */ From 9d991212cb4da219257964b0d20aaf57a194e558 Mon Sep 17 00:00:00 2001 From: aditkumar Date: Tue, 20 Mar 2018 10:52:52 -0400 Subject: [PATCH 12/32] line < 100 --- .../sql/catalyst/expressions/datetimeExpressions.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index b0e7a4a00e7d5..f7f3680002240 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -1118,10 +1118,10 @@ case class AddMonths(startDate: Expression, numMonths: Expression) /** * Returns number of months between dates `timestamp1` and `timestamp2`. * If `timestamp` is later than `timestamp2`, then the result is positive. - * If `timestamp1` and `timestamp2` are on the same day of month, or both are the last day of month, - * returns an integer (time of day will be ignored). - * Otherwise, the difference is calculated based on 31 days per month, and rounded to - * 8 digits. + * If `timestamp1` and `timestamp2` are on the same day of month, or both + * are the last day of month, returns an integer (time of day will be ignored). + * Otherwise, the difference is calculated based on 31 days per month, and + * rounded to 8 digits. */ // scalastyle:off line.size.limit @ExpressionDescription( From 682bc8cd766c405d241bf76af62ab03ac3cc26d2 Mon Sep 17 00:00:00 2001 From: aditkumar Date: Tue, 20 Mar 2018 11:00:29 -0400 Subject: [PATCH 13/32] python whitespace --- python/pyspark/sql/functions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py index 4e7602789fbd1..8fed5f3a66d3f 100644 --- a/python/pyspark/sql/functions.py +++ b/python/pyspark/sql/functions.py @@ -1078,13 +1078,13 @@ def add_months(start, months): @since(1.5) def months_between(date1, date2): """ - Returns number of months between dates date1 and date2. + Returns number of months between dates date1 and date2. If date1 is later than date2, then the result is positive. If date1 and date2 are on the same day of month, or both are the last day of month, returns an integer (time of day will be ignored). - + Otherwise, the difference is calculated based on 31 days per month, and rounded to - 8 digits. + 8 digits. >>> df = spark.createDataFrame([('1997-02-28 10:30:00', '1996-10-30')], ['date1', 'date2']) >>> df.select(months_between(df.date1, df.date2).alias('months')).collect() From 56b699f7a8eb805d734919f4f8c4b08b32b55ff0 Mon Sep 17 00:00:00 2001 From: aditkumar Date: Thu, 22 Mar 2018 08:51:24 -0400 Subject: [PATCH 14/32] Update DateTimeUtils.scala --- .../org/apache/spark/sql/catalyst/util/DateTimeUtils.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala index 0d6f9e9d9293f..750efc4b6de7d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala @@ -868,7 +868,7 @@ object DateTimeUtils { * microseconds since 1.1.1970. * * If time1 and time2 are on the same day of month, or both are the last day of month, - * returns an integer (time under a day will be ignored). + * time of a day will be ignored. * * Otherwise, the difference is calculated based on 31 days per month, and rounded to * 8 digits. @@ -882,7 +882,7 @@ object DateTimeUtils { * microseconds since 1.1.1970. * * If time1 and time2 are on the same day of month, or both are the last day of month, - * returns an integer (time under a day will be ignored). + * returns, time of day will be ignored. * * Otherwise, the difference is calculated based on 31 days per month, and rounded to * 8 digits. From d330efbcc0ee2be59eee5e8e99c585ed07fa77cd Mon Sep 17 00:00:00 2001 From: aditkumar Date: Thu, 22 Mar 2018 08:54:54 -0400 Subject: [PATCH 15/32] Reverting removing line --- .../spark/sql/catalyst/expressions/datetimeExpressions.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index f7f3680002240..36c527ce6633b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -1115,6 +1115,7 @@ case class AddMonths(startDate: Expression, numMonths: Expression) override def prettyName: String = "add_months" } + /** * Returns number of months between dates `timestamp1` and `timestamp2`. * If `timestamp` is later than `timestamp2`, then the result is positive. From 28ce561f41ec33434c25f89d6753755656230ec4 Mon Sep 17 00:00:00 2001 From: aditkumar Date: Thu, 22 Mar 2018 09:00:31 -0400 Subject: [PATCH 16/32] update description and comment style --- .../expressions/datetimeExpressions.scala | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 36c527ce6633b..b56c5a9937927 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -1117,16 +1117,21 @@ case class AddMonths(startDate: Expression, numMonths: Expression) } /** - * Returns number of months between dates `timestamp1` and `timestamp2`. - * If `timestamp` is later than `timestamp2`, then the result is positive. - * If `timestamp1` and `timestamp2` are on the same day of month, or both - * are the last day of month, returns an integer (time of day will be ignored). - * Otherwise, the difference is calculated based on 31 days per month, and - * rounded to 8 digits. -*/ + * Returns number of months between dates `timestamp1` and `timestamp2`. + * If `timestamp` is later than `timestamp2`, then the result is positive. + * If `timestamp1` and `timestamp2` are on the same day of month, or both + * are the last day of month, time of day will be ignored. + * Otherwise, the difference is calculated based on 31 days per month, and + * rounded to 8 digits. + */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(timestamp1, timestamp2) - Returns number of months between `timestamp1` and `timestamp2`. Positive if `timestamp1` is later than `timestamp2`", + usage = "_FUNC_(timestamp1, timestamp2) - + If `timestamp` is later than `timestamp2`, then the result is positive. + If `timestamp1` and `timestamp2` are on the same day of month, or both + are the last day of month, time of day will be ignored. + Otherwise, the difference is calculated based on 31 days per month, and + rounded to 8 digits. examples = """ Examples: > SELECT _FUNC_('1997-02-28 10:30:00', '1996-10-30'); From 0b115442576dc53db7452681d7b2ec152a3ee7da Mon Sep 17 00:00:00 2001 From: aditkumar Date: Thu, 22 Mar 2018 09:01:45 -0400 Subject: [PATCH 17/32] update description --- R/pkg/R/functions.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/pkg/R/functions.R b/R/pkg/R/functions.R index fc2cba68fd82b..8f711762226ff 100644 --- a/R/pkg/R/functions.R +++ b/R/pkg/R/functions.R @@ -1961,7 +1961,7 @@ setMethod("levenshtein", signature(y = "Column"), #' \code{months_between}: Returns number of months between dates \code{y} and \code{x}. #' If \code{y} is later than \code{x}, then the result is positive. #' If \code{y} and \code{x} are on the same day of month, or both are the last day of month, -#' returns an integer (time of day will be ignored). +#' time of day will be ignored. #' Otherwise, the difference is calculated based on 31 days per month, and rounded to #' 8 digits. #' @rdname column_datetime_diff_functions From ef43d7b747e5252af1f91e74e0b4e9c4cefef598 Mon Sep 17 00:00:00 2001 From: aditkumar Date: Thu, 22 Mar 2018 09:02:46 -0400 Subject: [PATCH 18/32] update description --- sql/core/src/main/scala/org/apache/spark/sql/functions.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala index 1d03d86a2c465..32fc2df14f75c 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala @@ -2711,7 +2711,7 @@ object functions { * Returns number of months between dates `date1` and `date2`. * If `date1` is later than `date2`, then the result is positive. * If `date1` and `date2` are on the same day of month, or both are the last day of month, - * returns an integer (time of day will be ignored). + * time of day will be ignored. * * Otherwise, the difference is calculated based on 31 days per month, and rounded to * 8 digits. From 292508deed398b318bb26d6df3c2133319634b1d Mon Sep 17 00:00:00 2001 From: aditkumar Date: Thu, 22 Mar 2018 09:09:34 -0400 Subject: [PATCH 19/32] ExpressionDescription --- .../catalyst/expressions/datetimeExpressions.scala | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index b56c5a9937927..9517372afacb9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -1126,12 +1126,14 @@ case class AddMonths(startDate: Expression, numMonths: Expression) */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(timestamp1, timestamp2) - - If `timestamp` is later than `timestamp2`, then the result is positive. - If `timestamp1` and `timestamp2` are on the same day of month, or both - are the last day of month, time of day will be ignored. - Otherwise, the difference is calculated based on 31 days per month, and - rounded to 8 digits. + usage = """ + _FUNC_(timestamp1, timestamp2) - + If `timestamp` is later than `timestamp2`, then the result is positive. + If `timestamp1` and `timestamp2` are on the same day of month, or both + are the last day of month, time of day will be ignored. + Otherwise, the difference is calculated based on 31 days per month, and + rounded to 8 digits. + """, examples = """ Examples: > SELECT _FUNC_('1997-02-28 10:30:00', '1996-10-30'); From 5d01c35aa20fe4173e513e2485a04e33f981ec77 Mon Sep 17 00:00:00 2001 From: aditkumar Date: Tue, 27 Mar 2018 17:10:46 -0400 Subject: [PATCH 20/32] Removing new line --- .../catalyst/expressions/datetimeExpressions.scala | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 9517372afacb9..9ad9821ffd680 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -1120,9 +1120,9 @@ case class AddMonths(startDate: Expression, numMonths: Expression) * Returns number of months between dates `timestamp1` and `timestamp2`. * If `timestamp` is later than `timestamp2`, then the result is positive. * If `timestamp1` and `timestamp2` are on the same day of month, or both - * are the last day of month, time of day will be ignored. - * Otherwise, the difference is calculated based on 31 days per month, and - * rounded to 8 digits. + * are the last day of month, time of day will be ignored. Otherwise, the + * difference is calculated based on 31 days per month, and rounded to + * 8 digits. */ // scalastyle:off line.size.limit @ExpressionDescription( @@ -1130,10 +1130,9 @@ case class AddMonths(startDate: Expression, numMonths: Expression) _FUNC_(timestamp1, timestamp2) - If `timestamp` is later than `timestamp2`, then the result is positive. If `timestamp1` and `timestamp2` are on the same day of month, or both - are the last day of month, time of day will be ignored. - Otherwise, the difference is calculated based on 31 days per month, and - rounded to 8 digits. - """, + are the last day of month, time of day will be ignored. Otherwise, the + difference is calculated based on 31 days per month, and rounded to 8 digits. + """, examples = """ Examples: > SELECT _FUNC_('1997-02-28 10:30:00', '1996-10-30'); From e74f25997c5e56938d849d40190ecf9b967c55ad Mon Sep 17 00:00:00 2001 From: aditkumar Date: Tue, 27 Mar 2018 17:16:58 -0400 Subject: [PATCH 21/32] adding back new line --- R/pkg/R/functions.R | 1 + 1 file changed, 1 insertion(+) diff --git a/R/pkg/R/functions.R b/R/pkg/R/functions.R index 8f711762226ff..7ec88c13e02a2 100644 --- a/R/pkg/R/functions.R +++ b/R/pkg/R/functions.R @@ -1958,6 +1958,7 @@ setMethod("levenshtein", signature(y = "Column"), }) #' @details +#' #' \code{months_between}: Returns number of months between dates \code{y} and \code{x}. #' If \code{y} is later than \code{x}, then the result is positive. #' If \code{y} and \code{x} are on the same day of month, or both are the last day of month, From ed369fff82eeaf2fc66e73b001cd2975d25a95b0 Mon Sep 17 00:00:00 2001 From: aditkumar Date: Tue, 27 Mar 2018 17:18:14 -0400 Subject: [PATCH 22/32] remove whitespace --- R/pkg/R/functions.R | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/R/pkg/R/functions.R b/R/pkg/R/functions.R index 7ec88c13e02a2..e96542eee5ec0 100644 --- a/R/pkg/R/functions.R +++ b/R/pkg/R/functions.R @@ -1960,11 +1960,9 @@ setMethod("levenshtein", signature(y = "Column"), #' @details #' #' \code{months_between}: Returns number of months between dates \code{y} and \code{x}. -#' If \code{y} is later than \code{x}, then the result is positive. -#' If \code{y} and \code{x} are on the same day of month, or both are the last day of month, -#' time of day will be ignored. -#' Otherwise, the difference is calculated based on 31 days per month, and rounded to -#' 8 digits. +#' If \code{y} is later than \code{x}, then the result is positive. If \code{y} and \code{x} +#' are on the same day of month, or both are the last day of month, time of day will be ignored. +#' Otherwise, the difference is calculated based on 31 days per month, and rounded to 8 digits. #' @rdname column_datetime_diff_functions #' @aliases months_between months_between,Column-method #' @note months_between since 1.5.0 From 3ca99489a1a88d16f371091e86e4a58962e00759 Mon Sep 17 00:00:00 2001 From: aditkumar Date: Mon, 16 Apr 2018 19:50:19 -0400 Subject: [PATCH 23/32] trying to make formatting consistent? --- .../sql/catalyst/expressions/datetimeExpressions.scala | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 9ad9821ffd680..8ba2e756dbc42 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -1127,10 +1127,9 @@ case class AddMonths(startDate: Expression, numMonths: Expression) // scalastyle:off line.size.limit @ExpressionDescription( usage = """ - _FUNC_(timestamp1, timestamp2) - - If `timestamp` is later than `timestamp2`, then the result is positive. - If `timestamp1` and `timestamp2` are on the same day of month, or both - are the last day of month, time of day will be ignored. Otherwise, the + _FUNC_(timestamp1, timestamp2) - If `timestamp` is later than `timestamp2`, then the result + is positive. If `timestamp1` and `timestamp2` are on the same day of month, or both + are the last day of month, time of day will be ignored. Otherwise, the difference is calculated based on 31 days per month, and rounded to 8 digits. """, examples = """ From 913eed849d218a34d6e28a2500301320ba21cecc Mon Sep 17 00:00:00 2001 From: aditkumar Date: Tue, 17 Apr 2018 12:20:59 -0400 Subject: [PATCH 24/32] empty line --- R/pkg/R/functions.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/pkg/R/functions.R b/R/pkg/R/functions.R index e96542eee5ec0..e1e103dd2f961 100644 --- a/R/pkg/R/functions.R +++ b/R/pkg/R/functions.R @@ -1958,11 +1958,11 @@ setMethod("levenshtein", signature(y = "Column"), }) #' @details -#' #' \code{months_between}: Returns number of months between dates \code{y} and \code{x}. #' If \code{y} is later than \code{x}, then the result is positive. If \code{y} and \code{x} #' are on the same day of month, or both are the last day of month, time of day will be ignored. #' Otherwise, the difference is calculated based on 31 days per month, and rounded to 8 digits. +#' #' @rdname column_datetime_diff_functions #' @aliases months_between months_between,Column-method #' @note months_between since 1.5.0 From e30473dcca87292c6aa6bb78920d7bcaadd636fe Mon Sep 17 00:00:00 2001 From: aditkumar Date: Fri, 20 Apr 2018 10:42:24 -0400 Subject: [PATCH 25/32] formatting @HyukjinKwon I think this matches yours? --- .../sql/catalyst/expressions/datetimeExpressions.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 8ba2e756dbc42..a58f88707cab4 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -1126,12 +1126,12 @@ case class AddMonths(startDate: Expression, numMonths: Expression) */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = """ + usage = """ _FUNC_(timestamp1, timestamp2) - If `timestamp` is later than `timestamp2`, then the result - is positive. If `timestamp1` and `timestamp2` are on the same day of month, or both - are the last day of month, time of day will be ignored. Otherwise, the - difference is calculated based on 31 days per month, and rounded to 8 digits. - """, + is positive. If `timestamp1` and `timestamp2` are on the same day of month, or both + are the last day of month, time of day will be ignored. Otherwise, the difference is + calculated based on 31 days per month, and rounded to 8 digits. + """, examples = """ Examples: > SELECT _FUNC_('1997-02-28 10:30:00', '1996-10-30'); From 6aa96ad8fbc7bc561fce2324587c71945e18df52 Mon Sep 17 00:00:00 2001 From: aditkumar Date: Fri, 20 Apr 2018 10:54:00 -0400 Subject: [PATCH 26/32] tabs > spaces --- .../spark/sql/catalyst/expressions/datetimeExpressions.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index a58f88707cab4..012061a110dd9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -1126,12 +1126,12 @@ case class AddMonths(startDate: Expression, numMonths: Expression) */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = """ + usage = """ _FUNC_(timestamp1, timestamp2) - If `timestamp` is later than `timestamp2`, then the result is positive. If `timestamp1` and `timestamp2` are on the same day of month, or both are the last day of month, time of day will be ignored. Otherwise, the difference is calculated based on 31 days per month, and rounded to 8 digits. - """, + """, examples = """ Examples: > SELECT _FUNC_('1997-02-28 10:30:00', '1996-10-30'); From 737aa0231654073ac42b02b26474d1a3ace76706 Mon Sep 17 00:00:00 2001 From: aditkumar Date: Thu, 3 May 2018 13:52:01 -0400 Subject: [PATCH 27/32] timestamp -> date for consistency --- .../sql/catalyst/expressions/datetimeExpressions.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 012061a110dd9..c341dc18a45fb 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -1117,9 +1117,9 @@ case class AddMonths(startDate: Expression, numMonths: Expression) } /** - * Returns number of months between dates `timestamp1` and `timestamp2`. - * If `timestamp` is later than `timestamp2`, then the result is positive. - * If `timestamp1` and `timestamp2` are on the same day of month, or both + * Returns number of months between dates `date1` and `date2`. + * If `date1` is later than `date2`, then the result is positive. + * If `date1` and `date2` are on the same day of month, or both * are the last day of month, time of day will be ignored. Otherwise, the * difference is calculated based on 31 days per month, and rounded to * 8 digits. @@ -1127,8 +1127,8 @@ case class AddMonths(startDate: Expression, numMonths: Expression) // scalastyle:off line.size.limit @ExpressionDescription( usage = """ - _FUNC_(timestamp1, timestamp2) - If `timestamp` is later than `timestamp2`, then the result - is positive. If `timestamp1` and `timestamp2` are on the same day of month, or both + _FUNC_(date1, date2) - If `date1` is later than `date2`, then the result + is positive. If `date1` and `date2` are on the same day of month, or both are the last day of month, time of day will be ignored. Otherwise, the difference is calculated based on 31 days per month, and rounded to 8 digits. """, From 31a126f4a6d3be102503639c6d09419a962e1f98 Mon Sep 17 00:00:00 2001 From: aditkumar Date: Thu, 3 May 2018 14:25:13 -0400 Subject: [PATCH 28/32] wording --- .../org/apache/spark/sql/catalyst/util/DateTimeUtils.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala index 4e984a0b0d76a..89895d575f840 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala @@ -905,7 +905,7 @@ object DateTimeUtils { * returns, time of day will be ignored. * * Otherwise, the difference is calculated based on 31 days per month. - * If `roundOff` is set to true, the result is rounded to 8 decimal places. + * The result is rounded to 8 decimal places if `roundOff` is set to true. */ def monthsBetween( time1: SQLTimestamp, From 22e943574e718667629588cc0a236f8952d088c7 Mon Sep 17 00:00:00 2001 From: Adit Kumar Date: Fri, 4 May 2018 14:10:30 -0400 Subject: [PATCH 29/32] removing overloaded/redundant monthsBetween --- .../spark/sql/catalyst/util/DateTimeUtils.scala | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala index 89895d575f840..086a6983a8a35 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala @@ -883,20 +883,6 @@ object DateTimeUtils { microseconds } - /** - * Returns number of months between time1 and time2. time1 and time2 are expressed in - * microseconds since 1.1.1970. - * - * If time1 and time2 are on the same day of month, or both are the last day of month, - * time of a day will be ignored. - * - * Otherwise, the difference is calculated based on 31 days per month, and rounded to - * 8 digits. - */ - def monthsBetween(time1: SQLTimestamp, time2: SQLTimestamp): Double = { - monthsBetween(time1, time2, defaultTimeZone()) - } - /** * Returns number of months between time1 and time2. time1 and time2 are expressed in * microseconds since 1.1.1970. From d571989ba8ebdce0d5742132bf35c9499e74d8e9 Mon Sep 17 00:00:00 2001 From: aditkumar Date: Mon, 7 May 2018 10:14:50 -0400 Subject: [PATCH 30/32] Update datetimeExpressions.scala --- .../spark/sql/catalyst/expressions/datetimeExpressions.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index f389c37812682..cabc4ccb7a3d2 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -1204,7 +1204,7 @@ case class AddMonths(startDate: Expression, numMonths: Expression) // scalastyle:off line.size.limit @ExpressionDescription( usage = """ - _FUNC_(date1, date2) - If `date1` is later than `date2`, then the result + _FUNC_(date1, date2[, roundOff]) - If `date1` is later than `date2`, then the result is positive. If `date1` and `date2` are on the same day of month, or both are the last day of month, time of day will be ignored. Otherwise, the difference is calculated based on 31 days per month, and rounded to 8 digits unless roundOff=false. From 6a3ba4af43072b2e135855e1c04f6b2987cd94aa Mon Sep 17 00:00:00 2001 From: aditkumar Date: Mon, 7 May 2018 10:16:52 -0400 Subject: [PATCH 31/32] Update DateTimeUtils.scala --- .../org/apache/spark/sql/catalyst/util/DateTimeUtils.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala index 086a6983a8a35..80f15053005ff 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala @@ -885,7 +885,7 @@ object DateTimeUtils { /** * Returns number of months between time1 and time2. time1 and time2 are expressed in - * microseconds since 1.1.1970. + * microseconds since 1.1.1970. If time1 is later than time2, the result is positive. * * If time1 and time2 are on the same day of month, or both are the last day of month, * returns, time of day will be ignored. From accf9ff6368efc5bbbe42681fa89f79409ddd87a Mon Sep 17 00:00:00 2001 From: aditkumar Date: Tue, 8 May 2018 13:10:55 -0400 Subject: [PATCH 32/32] date -> timestamp --- .../sql/catalyst/expressions/datetimeExpressions.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index cabc4ccb7a3d2..03422fecb3209 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -1194,9 +1194,9 @@ case class AddMonths(startDate: Expression, numMonths: Expression) } /** - * Returns number of months between dates `date1` and `date2`. - * If `date1` is later than `date2`, then the result is positive. - * If `date1` and `date2` are on the same day of month, or both + * Returns number of months between times `timestamp1` and `timestamp2`. + * If `timestamp1` is later than `timestamp2`, then the result is positive. + * If `timestamp1` and `timestamp2` are on the same day of month, or both * are the last day of month, time of day will be ignored. Otherwise, the * difference is calculated based on 31 days per month, and rounded to * 8 digits unless roundOff=false. @@ -1204,8 +1204,8 @@ case class AddMonths(startDate: Expression, numMonths: Expression) // scalastyle:off line.size.limit @ExpressionDescription( usage = """ - _FUNC_(date1, date2[, roundOff]) - If `date1` is later than `date2`, then the result - is positive. If `date1` and `date2` are on the same day of month, or both + _FUNC_(timestamp1, timestamp2[, roundOff]) - If `timestamp1` is later than `timestamp2`, then the result + is positive. If `timestamp1` and `timestamp2` are on the same day of month, or both are the last day of month, time of day will be ignored. Otherwise, the difference is calculated based on 31 days per month, and rounded to 8 digits unless roundOff=false. """,