MDEV-32188 make TIMESTAMP use whole 32-bit unsigned range

This patch extends the timestamp from
2038-01-19 03:14:07.999999 to 2106-02-07 06:28:15.999999
for 64 bit hardware and OS where 'long' is 64 bits.
This is true for 64 bit Linux but not for Windows.

This is done by treating the 32 bit stored int as unsigned instead of
signed.  This is safe as MariaDB has never accepted dates before the epoch
(1970).
The benefit of this approach that for normal timestamp the storage is
compatible with earlier version.

However for tables using system versioning we before stored a
timestamp with the year 2038 as the 'max timestamp', which is used to
detect current values.  This patch stores the new 2106 year max value
as the max timestamp. This means that old tables using system
versioning needs to be updated with mariadb-upgrade when moving them
to 11.4. That will be done in a separate commit.

Author: montywi

extra/mariabackup/xtrabackup.cc

@@ -2404,7 +2404,7 @@ static bool innodb_init_param()
 
 	/* Check that values don't overflow on 32-bit systems. */
 	if (sizeof(ulint) == 4) {
-          if (xtrabackup_use_memory > (longlong) UINT_MAX32) {
+		if (xtrabackup_use_memory > (longlong) UINT_MAX32) {
 			msg("mariabackup: use-memory can't be over 4GB"
 			    " on 32-bit systems");
 		}

include/my_global.h

@@ -782,7 +782,7 @@ inline unsigned long long my_double2ulonglong(double d)
 #define INT_MAX64       0x7FFFFFFFFFFFFFFFLL
 #define INT_MIN32       (~0x7FFFFFFFL)
 #define INT_MAX32       0x7FFFFFFFL
-#define UINT_MAX32      0xFFFFFFFFL
+#define UINT_MAX32      0xFFFFFFFFUL
 #define INT_MIN24       (~0x007FFFFF)
 #define INT_MAX24       0x007FFFFF
 #define UINT_MAX24      0x00FFFFFF

include/my_time.h

@@ -30,15 +30,26 @@ C_MODE_START
 extern MYSQL_PLUGIN_IMPORT ulonglong log_10_int[20];
 extern uchar days_in_month[];
 
+#if SIZEOF_LONG == 4
 #define MY_TIME_T_MAX LONG_MAX
 #define MY_TIME_T_MIN LONG_MIN
-
-/* Time handling defaults */
 #define TIMESTAMP_MAX_YEAR 2038
-#define TIMESTAMP_MIN_YEAR (1900 + YY_PART_YEAR - 1)
+#define TIMESTAMP_MAX_MONTH 1
+#define TIMESTAMP_MAX_DAY 19
 #define TIMESTAMP_MAX_VALUE INT_MAX32
 #define TIMESTAMP_MIN_VALUE 0
+#else
+/* Use 4 byte unsigned timestamp */
+#define MY_TIME_T_MAX ((longlong) UINT_MAX32)
+#define MY_TIME_T_MIN 0
+#define TIMESTAMP_MAX_YEAR 2106
+#define TIMESTAMP_MIN_VALUE 0
+#define TIMESTAMP_MAX_VALUE ((longlong) UINT_MAX32)
+#define TIMESTAMP_MAX_MONTH 2
+#define TIMESTAMP_MAX_DAY 7
+#endif /* SIZEOF_LONG */
 
+#define TIMESTAMP_MIN_YEAR (1900 + YY_PART_YEAR - 1)
 /* two-digit years < this are 20..; >= this are 19.. */
 #define YY_PART_YEAR	   70
 
@@ -48,8 +59,8 @@ extern uchar days_in_month[];
 */
 #if SIZEOF_TIME_T > 4 || defined(TIME_T_UNSIGNED)
 # define IS_TIME_T_VALID_FOR_TIMESTAMP(x) \
-    ((x) <= TIMESTAMP_MAX_VALUE && \
-     (x) >= TIMESTAMP_MIN_VALUE)
+  ((ulonglong) (x) <= TIMESTAMP_MAX_VALUE &&     \
+      ((x) >= TIMESTAMP_MIN_VALUE)
 #else
 # define IS_TIME_T_VALID_FOR_TIMESTAMP(x) \
     ((x) >= TIMESTAMP_MIN_VALUE)
@@ -181,7 +192,6 @@ void my_init_time(void);
 static inline my_bool validate_timestamp_range(const MYSQL_TIME *t)
 {
   if ((t->year > TIMESTAMP_MAX_YEAR || t->year < TIMESTAMP_MIN_YEAR) ||
-      (t->year == TIMESTAMP_MAX_YEAR && (t->month > 1 || t->day > 19)) ||
       (t->year == TIMESTAMP_MIN_YEAR && (t->month < 12 || t->day < 31)))
     return FALSE;
 

include/mysql_time.h

@@ -19,15 +19,14 @@
 
 /*
   Portable time_t replacement.
-  Should be signed and hold seconds for 1902 -- 2038-01-19 range
-  i.e at least a 32bit variable
+  For 32 bit systems holds seconds for 1970 - 2038-01-19 range
+  For 64 bit systems (where long is 64 bit) holds seconds for 1970 - 2106
 
   Using the system built in time_t is not an option as
   we rely on the above requirements in the time functions   
 */
 typedef long my_time_t;
 
-
 /*
   Time declarations shared between the server and client API:
   you should not add anything to this header unless it's used

libmysqld/libmysql.c

@@ -1314,8 +1314,8 @@ static my_bool reset_stmt_handle(MYSQL_STMT *stmt, uint flags);
 /* A macro to check truncation errors */
 
 #define IS_TRUNCATED(value, is_unsigned, min, max, umax) \
-        ((is_unsigned) ? (((value) > (umax) || (value) < 0) ? 1 : 0) : \
-                         (((value) > (max)  || (value) < (min)) ? 1 : 0))
+  ((is_unsigned) ? (( (value) > (umax)) ? 1 : 0) : \
+                    (((value) > (max)  || (value) < (min)) ? 1 : 0))
 
 #define BIND_RESULT_DONE 1
 /*

mysql-test/include/grep.inc

@@ -3,14 +3,22 @@
 # Usage:
 # --let $grep_file= /path/to/your/file
 # --let $grep_regex= your_regex_string
+# --let $grep_filter= filter
 # --source include/grep.inc
 
+
 --perl
   open (my $fh, "<", "$ENV{grep_file}") or die $!;
   while (<$fh>)
   {
-    /$ENV{grep_regex}/ &&
+    if  (/$ENV{grep_regex}/)
+    {
+      if ($ENV{grep_filter})
+      {
+        eval($ENV{grep_filter});
+      }
       print;
+    }
   }
   close $fh;
 EOF

mysql-test/include/have_32bit_timestamp.inc

@@ -0,0 +1,7 @@
+disable_query_log;
+disable_warnings;
+if (`SELECT from_unixtime((1<<31)+24*3600) is not null`) {
+--skip Need a 32 bit timestamps
+}
+enable_warnings;
+enable_query_log;

mysql-test/include/have_64bit_timestamp.inc

@@ -0,0 +1,7 @@
+disable_query_log;
+disable_warnings;
+if (`SELECT from_unixtime((1<<31)+24*3600) is null`) {
+--skip Need a 64 bit timestamps
+}
+enable_warnings;
+enable_query_log;

mysql-test/include/show_events.inc

@@ -117,6 +117,8 @@ let $script=
   s{SONAME ".*"}{SONAME "LIB"};
   s{DOLLARmysqltest_vardir}{MYSQLTEST_VARDIR}g;
   s{$binlog_database}{database};
+  s{2147483647.999999}{MAX_TIMESTAMP};
+  s{4294967295.999999}{MAX_TIMESTAMP};
   $modify_binlog_name
 ||
 

mysql-test/main/alter_table_online_debug.test

@@ -582,6 +582,7 @@ set debug_sync= 'now SIGNAL end';
 --connection default
 --reap
 show create table t1;
+--replace_result 4294967295 2147483647
 select *, UNIX_TIMESTAMP(row_start), UNIX_TIMESTAMP(row_end) from t1 for system_time all;
 
 ## at the moment DROP SYSTEM VERSIONING cannot be done online

mysql-test/main/func_time.result

@@ -581,37 +581,18 @@ NULL
 select from_unixtime(2147483647);
 from_unixtime(2147483647)
 2038-01-19 06:14:07
-select from_unixtime(2147483648);
-from_unixtime(2147483648)
-NULL
-Warnings:
-Warning	1292	Truncated incorrect unixtime value: '2147483648'
 select from_unixtime(0);
 from_unixtime(0)
 1970-01-01 03:00:00
 select unix_timestamp(from_unixtime(2147483647));
 unix_timestamp(from_unixtime(2147483647))
 2147483647
-select unix_timestamp(from_unixtime(2147483648));
-unix_timestamp(from_unixtime(2147483648))
-NULL
-Warnings:
-Warning	1292	Truncated incorrect unixtime value: '2147483648'
-select unix_timestamp('2039-01-20 01:00:00');
-unix_timestamp('2039-01-20 01:00:00')
-NULL
 select unix_timestamp('1968-01-20 01:00:00');
 unix_timestamp('1968-01-20 01:00:00')
 NULL
-select unix_timestamp('2038-02-10 01:00:00');
-unix_timestamp('2038-02-10 01:00:00')
-NULL
 select unix_timestamp('1969-11-20 01:00:00');
 unix_timestamp('1969-11-20 01:00:00')
 NULL
-select unix_timestamp('2038-01-20 01:00:00');
-unix_timestamp('2038-01-20 01:00:00')
-NULL
 select unix_timestamp('1969-12-30 01:00:00');
 unix_timestamp('1969-12-30 01:00:00')
 NULL
@@ -621,9 +602,6 @@ unix_timestamp('2038-01-17 12:00:00')
 select unix_timestamp('1970-01-01 03:00:01');
 unix_timestamp('1970-01-01 03:00:01')
 1
-select unix_timestamp('2038-01-19 07:14:07');
-unix_timestamp('2038-01-19 07:14:07')
-NULL
 SELECT CHARSET(DAYNAME(19700101));
 CHARSET(DAYNAME(19700101))
 latin1

mysql-test/main/func_time.test

@@ -276,7 +276,6 @@ select unix_timestamp('1969-12-01 19:00:01');
 select from_unixtime(-1);
 # check for from_unixtime(2^31-1) and from_unixtime(2^31)
 select from_unixtime(2147483647);
-select from_unixtime(2147483648);
 select from_unixtime(0);
 
 #
@@ -285,18 +284,14 @@ select from_unixtime(0);
 # unix_timestamp are consistent, when working with boundary dates.
 #
 select unix_timestamp(from_unixtime(2147483647));
-select unix_timestamp(from_unixtime(2147483648));
 
 # check for invalid dates
 
 # bad year
-select unix_timestamp('2039-01-20 01:00:00');
 select unix_timestamp('1968-01-20 01:00:00');
 # bad month
-select unix_timestamp('2038-02-10 01:00:00');
 select unix_timestamp('1969-11-20 01:00:00');
 # bad day
-select unix_timestamp('2038-01-20 01:00:00');
 select unix_timestamp('1969-12-30 01:00:00');
 
 #
@@ -310,9 +305,6 @@ select unix_timestamp('2038-01-17 12:00:00');
 #
 select unix_timestamp('1970-01-01 03:00:01');
 
-# check bad date, close to the boundary (we cut them off in the very end)
-select unix_timestamp('2038-01-19 07:14:07');
-
 #
 # Bug #28759: DAYNAME() and MONTHNAME() return binary string
 #

mysql-test/main/func_time_32.result

@@ -0,0 +1,120 @@
+#
+# Test dates close to upper range
+#
+set time_zone="+03:00";
+select from_unixtime(2147483648);
+from_unixtime(2147483648)
+NULL
+Warnings:
+Warning	1292	Truncated incorrect unixtime value: '2147483648'
+select unix_timestamp(from_unixtime(2147483648));
+unix_timestamp(from_unixtime(2147483648))
+NULL
+Warnings:
+Warning	1292	Truncated incorrect unixtime value: '2147483648'
+# bad year
+select unix_timestamp('2039-01-20 01:00:00');
+unix_timestamp('2039-01-20 01:00:00')
+NULL
+# bad month
+select unix_timestamp('2038-02-10 01:00:00');
+unix_timestamp('2038-02-10 01:00:00')
+NULL
+# bad day
+select unix_timestamp('2038-01-20 01:00:00');
+unix_timestamp('2038-01-20 01:00:00')
+NULL
+# check bad date, close to the boundary (we cut them off in the very end)
+select unix_timestamp('2038-01-19 07:14:07');
+unix_timestamp('2038-01-19 07:14:07')
+NULL
+set time_zone=MET;
+select unix_timestamp('2038-01-19 04:14:07'),
+unix_timestamp('2038-01-19 04:14:08');
+unix_timestamp('2038-01-19 04:14:07')	unix_timestamp('2038-01-19 04:14:08')
+2147483647	NULL
+set time_zone= @@global.time_zone;
+#
+# Functions that construct DATETIME
+#
+SET time_zone='+00:00';
+SET sql_mode=IF(@@version LIKE '%MariaDB%', 'TIME_ROUND_FRACTIONAL', '');
+CREATE TABLE t1 (id SERIAL, a DECIMAL(30,10));
+INSERT INTO t1 (a) VALUES (2147483647.9999999);
+SELECT a, FROM_UNIXTIME(a) FROM t1 ORDER BY id;
+a	FROM_UNIXTIME(a)
+2147483647.9999999000	NULL
+DROP TABLE t1;
+set time_zone= @@global.time_zone;
+set sql_mode=default;
+#
+# Corner case:
+# ALTER TIMESTAMP to a shorter TIMESTAMP
+# All values round, maximum possible value truncates.
+#
+SET time_zone='+00:00';
+SET sql_mode=IF(@@version LIKE '%MariaDB%', 'TIME_ROUND_FRACTIONAL', '');
+CREATE TABLE t1 (ID INT, a TIMESTAMP(6), comment VARCHAR(64));
+INSERT INTO t1 VALUES (0, '2038-01-18 23:59:59.999999', 'Should round');
+INSERT INTO t1 VALUES (1, '2038-01-19 03:14:06.999999', 'Should round');
+INSERT INTO t1 VALUES (2, '2038-01-19 03:14:07.999999', 'Should truncate');
+ALTER TABLE t1 MODIFY a TIMESTAMP(5);
+Warnings:
+Warning	1264	Out of range value for column 'a' at row 3
+SELECT * FROM t1;
+ID	a	comment
+0	2038-01-19 00:00:00.00000	Should round
+1	2038-01-19 03:14:07.00000	Should round
+2	2038-01-19 03:14:07.99999	Should truncate
+DROP TABLE t1;
+set time_zone= @@global.time_zone;
+set sql_mode=default;
+
+# Let us test range for TIMESTAMP
+#
+create table t1 (ts timestamp);
+set time_zone='UTC';
+insert into t1 values ('2038-01-19 03:14:07'),('2038-01-19 03:14:08');
+Warnings:
+Warning	1264	Out of range value for column 'ts' at row 2
+select * from t1;
+ts
+2038-01-19 03:14:07
+0000-00-00 00:00:00
+truncate table t1;
+set time_zone='MET';
+insert into t1 values ('2038-01-19 04:14:07'),('2038-01-19 04:14:08');
+Warnings:
+Warning	1264	Out of range value for column 'ts' at row 2
+select * from t1;
+ts
+2038-01-19 04:14:07
+0000-00-00 00:00:00
+truncate table t1;
+set time_zone='+01:30';
+insert into t1 values ('2038-01-19 04:44:07'),('2038-01-19 04:44:08');
+Warnings:
+Warning	1264	Out of range value for column 'ts' at row 2
+select * from t1;
+ts
+2038-01-19 04:44:07
+0000-00-00 00:00:00
+drop table t1;
+SET time_zone='+00:00';
+CREATE TABLE t1 (ts0 TIMESTAMP, ts1 TIMESTAMP(1));
+INSERT INTO t1 VALUES ('2001-01-01 10:20:30', '2001-01-01 10:20:30.9');
+# Corner case
+UPDATE t1 SET ts1=FROM_UNIXTIME(2147483647.9);
+UPDATE t1 SET ts0=COALESCE(ts1);
+SELECT * FROM t1;
+ts0	ts1
+2038-01-19 03:14:07	2038-01-19 03:14:07.9
+DROP TABLE t1;
+SET time_zone=DEFAULT;
+# Let us test CONVERT_TZ function
+select convert_tz('2038-01-19 04:14:08', 'MET', 'UTC');
+convert_tz('2038-01-19 04:14:08', 'MET', 'UTC')
+2038-01-19 04:14:08
+select convert_tz('2103-01-01 04:00:00', 'MET', 'UTC');
+convert_tz('2103-01-01 04:00:00', 'MET', 'UTC')
+2103-01-01 04:00:00