[CH-219] Support rlike/regexp_replace/regexp_extract/coalesce/DATE_AD…

…D/DATE_SUB/DATEDIFF/modulo/concat_ws/collect_list/FROM_UNIXTIME (ClickHouse#230)

* support function remainder

* support collect_list

* support function from_unixtime partly

* support datediff

* support datediff

* Add support of Date32 arguments

* Add ToExtendedRelativeDayNumImpl

* Add transforms for other arguments lke year, quarter etc

* Add test 02457_datediff_via_unix_epoch

* Set UTC for 02457_datediff_via_unix_epoch

* Fix message about allowed argument types

* Fix Date32 argument in dispatchConstForSecondColumn

* Add 02458_datediff_date32 test

* Update documentation

* Use {} in error message formatting

* Add UTC for toDate32 in tests

* Add toStableRelativeHourNum

* Remove UTC from 02458_datediff_date32 and 02457_datediff_via_unix_epoch tests

* Remove toExtendedReplated; Add template argument is_extended_result

* Add toStableRelativeHourNum to gtest_DateLUTImpl.cpp

* Replace is_extended_result by ResultPrecision

* finish dev

* remove duplicated log

* finish dev

* support to_unixtimestamp and unixtimestamp function

Co-authored-by: Roman Vasin <r.vasin@arenadata.io>

docs/en/sql-reference/functions/date-time-functions.md

@@ -652,9 +652,9 @@ Aliases: `dateDiff`, `DATE_DIFF`.
     - `quarter`
     - `year`
 
--   `startdate` — The first time value to subtract (the subtrahend). [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-reference/data-types/datetime.md).
+-   `startdate` — The first time value to subtract (the subtrahend). [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md).
 
--   `enddate` — The second time value to subtract from (the minuend). [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-reference/data-types/datetime.md).
+-   `enddate` — The second time value to subtract from (the minuend). [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md).
 
 -   `timezone` — [Timezone name](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) (optional). If specified, it is applied to both `startdate` and `enddate`. If not specified, timezones of `startdate` and `enddate` are used. If they are not the same, the result is unspecified. [String](../../sql-reference/data-types/string.md).
 

docs/ru/sql-reference/functions/date-time-functions.md

@@ -701,9 +701,9 @@ date_diff('unit', startdate, enddate, [timezone])
     - `quarter`
     - `year`
 
--   `startdate` — первая дата или дата со временем, которая вычитается из `enddate`. [Date](../../sql-reference/data-types/date.md) или [DateTime](../../sql-reference/data-types/datetime.md).
+-   `startdate` — первая дата или дата со временем, которая вычитается из `enddate`. [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) или [DateTime64](../../sql-reference/data-types/datetime64.md).
 
--   `enddate` — вторая дата или дата со временем, из которой вычитается `startdate`. [Date](../../sql-reference/data-types/date.md) или [DateTime](../../sql-reference/data-types/datetime.md).
+-   `enddate` — вторая дата или дата со временем, из которой вычитается `startdate`. [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) или [DateTime64](../../sql-reference/data-types/datetime64.md).
 
 -   `timezone` — [часовой пояс](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) (необязательно). Если этот аргумент указан, то он применяется как для `startdate`, так и для `enddate`. Если этот аргумент не указан, то используются часовые пояса аргументов `startdate` и `enddate`. Если часовые пояса аргументов `startdate` и `enddate` не совпадают, то результат не определен. [String](../../sql-reference/data-types/string.md).
 

src/Common/DateLUTImpl.h

@@ -852,6 +852,19 @@ class DateLUTImpl
         return toRelativeHourNum(lut[toLUTIndex(v)].date);
     }
 
+    /// The same formula is used for positive time (after Unix epoch) and negative time (before Unix epoch).
+    /// It’s needed for correct work of dateDiff function.
+    inline Time toStableRelativeHourNum(Time t) const
+    {
+        return (t + DATE_LUT_ADD + 86400 - offset_at_start_of_epoch) / 3600 - (DATE_LUT_ADD / 3600);
+    }
+
+    template <typename DateOrTime>
+    inline Time toStableRelativeHourNum(DateOrTime v) const
+    {
+        return toStableRelativeHourNum(lut[toLUTIndex(v)].date);
+    }
+
     inline Time toRelativeMinuteNum(Time t) const /// NOLINT
     {
         return (t + DATE_LUT_ADD) / 60 - (DATE_LUT_ADD / 60);

src/Common/tests/gtest_DateLUTImpl.cpp

@@ -133,6 +133,7 @@ TEST(DateLUTTest, TimeValuesInMiddleOfRange)
     EXPECT_EQ(lut.toRelativeMonthNum(time), 24237 /*unsigned*/);
     EXPECT_EQ(lut.toRelativeQuarterNum(time), 8078 /*unsigned*/);
     EXPECT_EQ(lut.toRelativeHourNum(time), 435736 /*time_t*/);
+    EXPECT_EQ(lut.toStableRelativeHourNum(time), 435757 /*time_t*/);
     EXPECT_EQ(lut.toRelativeMinuteNum(time), 26144180 /*time_t*/);
     EXPECT_EQ(lut.toStartOfMinuteInterval(time, 6), 1568650680 /*time_t*/);
     EXPECT_EQ(lut.toStartOfSecondInterval(time, 7), 1568650811 /*time_t*/);
@@ -193,6 +194,7 @@ TEST(DateLUTTest, TimeValuesAtLeftBoderOfRange)
     EXPECT_EQ(lut.toRelativeMonthNum(time), 23641 /*unsigned*/); // ?
     EXPECT_EQ(lut.toRelativeQuarterNum(time), 7880 /*unsigned*/); // ?
     EXPECT_EQ(lut.toRelativeHourNum(time), 0 /*time_t*/);
+    EXPECT_EQ(lut.toStableRelativeHourNum(time), 24 /*time_t*/);
     EXPECT_EQ(lut.toRelativeMinuteNum(time), 0 /*time_t*/);
     EXPECT_EQ(lut.toStartOfMinuteInterval(time, 6), 0 /*time_t*/);
     EXPECT_EQ(lut.toStartOfSecondInterval(time, 7), 0 /*time_t*/);
@@ -254,6 +256,7 @@ TEST(DateLUTTest, TimeValuesAtRightBoderOfRangeOfOldLUT)
     EXPECT_EQ(lut.toRelativeMonthNum(time), 25273 /*unsigned*/);
     EXPECT_EQ(lut.toRelativeQuarterNum(time), 8424 /*unsigned*/);
     EXPECT_EQ(lut.toRelativeHourNum(time), 1192873 /*time_t*/);
+    EXPECT_EQ(lut.toStableRelativeHourNum(time), 1192897 /*time_t*/);
     EXPECT_EQ(lut.toRelativeMinuteNum(time), 71572397 /*time_t*/);
     EXPECT_EQ(lut.toStartOfMinuteInterval(time, 6), 4294343520 /*time_t*/);
     EXPECT_EQ(lut.toStartOfSecondInterval(time, 7), 4294343872 /*time_t*/);

src/Functions/DateTimeTransforms.h

@@ -754,21 +754,39 @@ struct ToISOWeekImpl
     using FactorTransform = ToISOYearImpl;
 };
 
+enum class ResultPrecision
+{
+    Standard,
+    Extended
+};
+
+/// Standard precision results (precision_ == ResultPrecision::Standard) potentially lead to overflows when returning values.
+/// This mode is used by SQL functions "toRelative*Num()" which cannot easily be changed due to backward compatibility.
+/// According to documentation, these functions merely need to compute the time difference to a deterministic, fixed point in the past.
+/// As a future TODO, we should fix their behavior in a backwards-compatible way.
+/// See https://github.com/ClickHouse/ClickHouse/issues/41977#issuecomment-1267536814.
+template <ResultPrecision precision_>
 struct ToRelativeYearNumImpl
 {
     static constexpr auto name = "toRelativeYearNum";
 
-    static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone)
+    static inline auto execute(Int64 t, const DateLUTImpl & time_zone)
     {
-        return time_zone.toYear(t);
+        if constexpr (precision_ == ResultPrecision::Extended)
+            return static_cast<Int16>(time_zone.toYear(t));
+        else
+            return static_cast<UInt16>(time_zone.toYear(t));
     }
     static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone)
     {
         return time_zone.toYear(static_cast<time_t>(t));
     }
-    static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
+    static inline auto execute(Int32 d, const DateLUTImpl & time_zone)
     {
-        return time_zone.toYear(ExtendedDayNum(d));
+        if constexpr (precision_ == ResultPrecision::Extended)
+            return static_cast<Int16>(time_zone.toYear(ExtendedDayNum(d)));
+        else
+            return static_cast<UInt16>(time_zone.toYear(ExtendedDayNum(d)));
     }
     static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
@@ -778,21 +796,28 @@ struct ToRelativeYearNumImpl
     using FactorTransform = ZeroTransform;
 };
 
+template <ResultPrecision precision_>
 struct ToRelativeQuarterNumImpl
 {
     static constexpr auto name = "toRelativeQuarterNum";
 
-    static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone)
+    static inline auto execute(Int64 t, const DateLUTImpl & time_zone)
     {
-        return time_zone.toRelativeQuarterNum(t);
+        if constexpr (precision_ == ResultPrecision::Extended)
+            return static_cast<Int32>(time_zone.toRelativeQuarterNum(t));
+        else
+            return static_cast<UInt16>(time_zone.toRelativeQuarterNum(t));
     }
     static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone)
     {
         return time_zone.toRelativeQuarterNum(static_cast<time_t>(t));
     }
-    static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
+    static inline auto execute(Int32 d, const DateLUTImpl & time_zone)
     {
-        return time_zone.toRelativeQuarterNum(ExtendedDayNum(d));
+        if constexpr (precision_ == ResultPrecision::Extended)
+            return static_cast<Int32>(time_zone.toRelativeQuarterNum(ExtendedDayNum(d)));
+        else
+            return static_cast<UInt16>(time_zone.toRelativeQuarterNum(ExtendedDayNum(d)));
     }
     static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
@@ -802,21 +827,28 @@ struct ToRelativeQuarterNumImpl
     using FactorTransform = ZeroTransform;
 };
 
+template <ResultPrecision precision_>
 struct ToRelativeMonthNumImpl
 {
     static constexpr auto name = "toRelativeMonthNum";
 
-    static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone)
+    static inline auto execute(Int64 t, const DateLUTImpl & time_zone)
     {
-        return time_zone.toRelativeMonthNum(t);
+        if constexpr (precision_ == ResultPrecision::Extended)
+            return static_cast<Int32>(time_zone.toRelativeMonthNum(t));
+        else
+            return static_cast<UInt16>(time_zone.toRelativeMonthNum(t));
     }
     static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone)
     {
         return time_zone.toRelativeMonthNum(static_cast<time_t>(t));
     }
-    static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
+    static inline auto execute(Int32 d, const DateLUTImpl & time_zone)
     {
-        return time_zone.toRelativeMonthNum(ExtendedDayNum(d));
+        if constexpr (precision_ == ResultPrecision::Extended)
+            return static_cast<Int32>(time_zone.toRelativeMonthNum(ExtendedDayNum(d)));
+        else
+            return static_cast<UInt16>(time_zone.toRelativeMonthNum(ExtendedDayNum(d)));
     }
     static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
@@ -826,21 +858,28 @@ struct ToRelativeMonthNumImpl
     using FactorTransform = ZeroTransform;
 };
 
+template <ResultPrecision precision_>
 struct ToRelativeWeekNumImpl
 {
     static constexpr auto name = "toRelativeWeekNum";
 
-    static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone)
+    static inline auto execute(Int64 t, const DateLUTImpl & time_zone)
     {
-        return time_zone.toRelativeWeekNum(t);
+        if constexpr (precision_ == ResultPrecision::Extended)
+            return static_cast<Int32>(time_zone.toRelativeWeekNum(t));
+        else
+            return static_cast<UInt16>(time_zone.toRelativeWeekNum(t));
     }
     static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone)
     {
         return time_zone.toRelativeWeekNum(static_cast<time_t>(t));
     }
-    static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
+    static inline auto execute(Int32 d, const DateLUTImpl & time_zone)
     {
-        return time_zone.toRelativeWeekNum(ExtendedDayNum(d));
+        if constexpr (precision_ == ResultPrecision::Extended)
+            return static_cast<Int32>(time_zone.toRelativeWeekNum(ExtendedDayNum(d)));
+        else
+            return static_cast<UInt16>(time_zone.toRelativeWeekNum(ExtendedDayNum(d)));
     }
     static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
@@ -850,21 +889,28 @@ struct ToRelativeWeekNumImpl
     using FactorTransform = ZeroTransform;
 };
 
+template <ResultPrecision precision_>
 struct ToRelativeDayNumImpl
 {
     static constexpr auto name = "toRelativeDayNum";
 
-    static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone)
+    static inline auto execute(Int64 t, const DateLUTImpl & time_zone)
     {
-        return time_zone.toDayNum(t);
+        if constexpr (precision_ == ResultPrecision::Extended)
+            return static_cast<Int64>(time_zone.toDayNum(t));
+        else
+            return static_cast<UInt16>(time_zone.toDayNum(t));
     }
     static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone)
     {
         return time_zone.toDayNum(static_cast<time_t>(t));
     }
-    static inline UInt16 execute(Int32 d, const DateLUTImpl &)
+    static inline auto execute(Int32 d, const DateLUTImpl &)
     {
-        return static_cast<ExtendedDayNum>(d);
+        if constexpr (precision_ == ResultPrecision::Extended)
+            return static_cast<Int32>(static_cast<ExtendedDayNum>(d));
+        else
+            return static_cast<UInt16>(static_cast<ExtendedDayNum>(d));
     }
     static inline UInt16 execute(UInt16 d, const DateLUTImpl &)
     {
@@ -874,46 +920,65 @@ struct ToRelativeDayNumImpl
     using FactorTransform = ZeroTransform;
 };
 
-
+template <ResultPrecision precision_>
 struct ToRelativeHourNumImpl
 {
     static constexpr auto name = "toRelativeHourNum";
 
-    static inline UInt32 execute(Int64 t, const DateLUTImpl & time_zone)
+    static inline auto execute(Int64 t, const DateLUTImpl & time_zone)
     {
-        return time_zone.toRelativeHourNum(t);
+        if constexpr (precision_ == ResultPrecision::Extended)
+            return static_cast<Int64>(time_zone.toStableRelativeHourNum(t));
+        else
+            return static_cast<UInt32>(time_zone.toRelativeHourNum(t));
     }
     static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone)
     {
-        return time_zone.toRelativeHourNum(static_cast<time_t>(t));
+        if constexpr (precision_ == ResultPrecision::Extended)
+            return time_zone.toStableRelativeHourNum(static_cast<time_t>(t));
+        else
+            return time_zone.toRelativeHourNum(static_cast<time_t>(t));
     }
-    static inline UInt32 execute(Int32 d, const DateLUTImpl & time_zone)
+    static inline auto execute(Int32 d, const DateLUTImpl & time_zone)
     {
-        return time_zone.toRelativeHourNum(ExtendedDayNum(d));
+        if constexpr (precision_ == ResultPrecision::Extended)
+            return static_cast<Int64>(time_zone.toStableRelativeHourNum(ExtendedDayNum(d)));
+        else
+            return static_cast<UInt32>(time_zone.toRelativeHourNum(ExtendedDayNum(d)));
     }
     static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
-        return time_zone.toRelativeHourNum(DayNum(d));
+        if constexpr (precision_ == ResultPrecision::Extended)
+            return time_zone.toStableRelativeHourNum(DayNum(d));
+        else
+            return time_zone.toRelativeHourNum(DayNum(d));
     }
 
     using FactorTransform = ZeroTransform;
 };
 
+template <ResultPrecision precision_>
 struct ToRelativeMinuteNumImpl
 {
     static constexpr auto name = "toRelativeMinuteNum";
 
-    static inline UInt32 execute(Int64 t, const DateLUTImpl & time_zone)
+    static inline auto execute(Int64 t, const DateLUTImpl & time_zone)
     {
-        return time_zone.toRelativeMinuteNum(t);
+        if constexpr (precision_ == ResultPrecision::Extended)
+            return static_cast<Int64>(time_zone.toRelativeMinuteNum(t));
+        else
+            return static_cast<UInt32>(time_zone.toRelativeMinuteNum(t));
     }
     static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone)
     {
         return time_zone.toRelativeMinuteNum(static_cast<time_t>(t));
     }
-    static inline UInt32 execute(Int32 d, const DateLUTImpl & time_zone)
+    static inline auto execute(Int32 d, const DateLUTImpl & time_zone)
     {
-        return time_zone.toRelativeMinuteNum(ExtendedDayNum(d));
+        if constexpr (precision_ == ResultPrecision::Extended)
+            return static_cast<Int64>(time_zone.toRelativeMinuteNum(ExtendedDayNum(d)));
+        else
+            return static_cast<UInt32>(time_zone.toRelativeMinuteNum(ExtendedDayNum(d)));
     }
     static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
@@ -923,6 +988,7 @@ struct ToRelativeMinuteNumImpl
     using FactorTransform = ZeroTransform;
 };
 
+template <ResultPrecision precision_>
 struct ToRelativeSecondNumImpl
 {
     static constexpr auto name = "toRelativeSecondNum";
@@ -935,9 +1001,12 @@ struct ToRelativeSecondNumImpl
     {
         return t;
     }
-    static inline UInt32 execute(Int32 d, const DateLUTImpl & time_zone)
+    static inline auto execute(Int32 d, const DateLUTImpl & time_zone)
     {
-        return time_zone.fromDayNum(ExtendedDayNum(d));
+        if constexpr (precision_ == ResultPrecision::Extended)
+            return static_cast<Int64>(time_zone.fromDayNum(ExtendedDayNum(d)));
+        else
+            return static_cast<UInt32>(time_zone.fromDayNum(ExtendedDayNum(d)));
     }
     static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
     {