ALTER TABLE and replication should convert old row_end timestamps to …

…new timestamp range

MDEV-32188 make TIMESTAMP use whole 32-bit unsigned range

- Added --update-history option to mariadb-dump to change 2038
  row_end timestamp to 2106.
- Updated ALTER TABLE ... to convert old row_end timestamps to
  2106 timestamp for tables created before MariaDB 11.4.0.
- Fixed bug in CHECK TABLE where we wrongly suggested to USE REPAIR
  TABLE when ALTER TABLE...FORCE is needed.
- mariadb-check printed table names that where used with REPAIR TABLE but
  did not print table names used with ALTER TABLE or with name repair.
  Fixed by always printing a table that is fixed if --silent is not
  used.
- Added TABLE::vers_fix_old_timestamp() that will change max-timestamp
  for versioned tables when replication from a pre-11.4.0 server.

A few test cases changed. This is caused by:
- CHECK TABLE now prints 'Please do ALTER TABLE... instead of
  'Please do REPAIR TABLE' when there is a problem with the information
  in the .frm file (for example a very old frm file).
- mariadb-check now prints repaired table names.
- mariadb-check also now prints nicer error message in case ALTER TABLE
  is needed to repair a table.

client/client_priv.h

@@ -72,6 +72,7 @@ enum options_client
   OPT_IGNORE_SERVER_IDS,
   OPT_DO_SERVER_IDS,
   OPT_SSL_FP, OPT_SSL_FPLIST,
+  OPT_UPDATE_HISTORY,
   OPT_MAX_CLIENT_OPTION /* should be always the last */
 };
 

client/mysqlcheck.c

@@ -801,7 +801,7 @@ static int fix_table_storage_name(const char *name)
               name, name + 9);
 
   rc= run_query(qbuf, 1);
-  if (verbose)
+  if (!opt_silent)
     printf("%-50s %s\n", name, rc ? "FAILED" : "OK");
   DBUG_RETURN(rc);
 }
@@ -817,7 +817,7 @@ static int fix_database_storage_name(const char *name)
   my_snprintf(qbuf, sizeof(qbuf), "ALTER DATABASE %`s UPGRADE DATA DIRECTORY "
               "NAME", name);
   rc= run_query(qbuf, 1);
-  if (verbose)
+  if (!opt_silent)
     printf("%-50s %s\n", name, rc ? "FAILED" : "OK");
   DBUG_RETURN(rc);
 }
@@ -840,8 +840,8 @@ static int rebuild_table(char *name)
     fprintf(stderr, "Error: %s\n", mysql_error(sock));
     rc= 1;
   }
-  if (verbose)
-    printf("%-50s %s\n", name, rc ? "FAILED" : "FIXED");
+  if (!opt_silent)
+    printf("%-50s %s\n", name, rc ? "FAILED" : "OK");
   my_free(query);
   DBUG_RETURN(rc);
 }
@@ -1035,7 +1035,6 @@ static void __attribute__((noinline)) print_result()
   MYSQL_RES *res;
   MYSQL_ROW row;
   char prev[(NAME_LEN+9)*3+2];
-  char prev_alter[MAX_ALTER_STR_SIZE];
   size_t length_of_db= strlen(sock->db);
   my_bool found_error=0, table_rebuild=0;
   DYNAMIC_ARRAY *array4repair= &tables4repair;
@@ -1044,7 +1043,6 @@ static void __attribute__((noinline)) print_result()
   res = mysql_use_result(sock);
 
   prev[0] = '\0';
-  prev_alter[0]= 0;
   while ((row = mysql_fetch_row(res)))
   {
     int changed = strcmp(prev, row[0]);
@@ -1061,19 +1059,13 @@ static void __attribute__((noinline)) print_result()
 	  strcmp(row[3],"OK"))
       {
         if (table_rebuild)
-        {
-          if (prev_alter[0])
-            insert_dynamic(&alter_table_cmds, (uchar*) prev_alter);
-          else
-            insert_table_name(&tables4rebuild, prev, length_of_db);
-        }
+          insert_table_name(&tables4rebuild, prev, length_of_db);
         else
           insert_table_name(array4repair, prev, length_of_db);
       }
       array4repair= &tables4repair;
       found_error=0;
       table_rebuild=0;
-      prev_alter[0]= 0;
       if (opt_silent)
 	continue;
     }
@@ -1083,20 +1075,28 @@ static void __attribute__((noinline)) print_result()
     {
       /*
         If the error message includes REPAIR TABLE, we assume it means
-        we have to run upgrade on it. In this case we write a nicer message
+        we have to run REPAIR on it. In this case we write a nicer message
         than "Please do "REPAIR TABLE""...
+        If the message inclused ALTER TABLE then there is something wrong
+        with the table definition and we have to run ALTER TABLE to fix it.
+        Write also a nice error message for this csae.
       */
       if (!strcmp(row[2],"error") && strstr(row[3],"REPAIR "))
       {
-        printf("%-50s %s", row[0], "Needs upgrade");
+        printf("%-50s %s", row[0], "Needs upgrade with REPAIR");
         array4repair= strstr(row[3], "VIEW") ? &views4repair : &tables4repair;
       }
+      else if (!strcmp(row[2],"error") && strstr(row[3],"ALTER TABLE"))
+      {
+        printf("%-50s %s", row[0], "Needs upgrade with ALTER TABLE FORCE");
+        array4repair= &tables4rebuild;
+      }
       else
         printf("%s\n%-9s: %s", row[0], row[2], row[3]);
-      if (opt_auto_repair && strcmp(row[2],"note"))
+      if (strcmp(row[2],"note"))
       {
         found_error=1;
-        if (opt_auto_repair && strstr(row[3], "ALTER TABLE") != NULL)
+        if (strstr(row[3], "ALTER TABLE"))
           table_rebuild=1;
       }
     }
@@ -1109,12 +1109,7 @@ static void __attribute__((noinline)) print_result()
   if (found_error && opt_auto_repair && what_to_do != DO_REPAIR)
   {
     if (table_rebuild)
-    {
-      if (prev_alter[0])
-        insert_dynamic(&alter_table_cmds, prev_alter);
-      else
-        insert_table_name(&tables4rebuild, prev, length_of_db);
-    }
+      insert_table_name(&tables4rebuild, prev, length_of_db);
     else
       insert_table_name(array4repair, prev, length_of_db);
   }

client/mysqldump.cc

@@ -129,7 +129,7 @@ static my_bool  verbose= 0, opt_no_create_info= 0, opt_no_data= 0, opt_no_data_m
                 opt_include_master_host_port= 0,
                 opt_events= 0, opt_comments_used= 0,
                 opt_alltspcs=0, opt_notspcs= 0, opt_logging,
-                opt_header=0,
+                opt_header=0, opt_update_history= 0,
                 opt_drop_trigger= 0, opt_dump_history= 0;
 #define OPT_SYSTEM_ALL 1
 #define OPT_SYSTEM_USERS 2
@@ -151,7 +151,8 @@ static my_bool insert_pat_inited= 0, debug_info_flag= 0, debug_check_flag= 0,
 static ulong opt_max_allowed_packet, opt_net_buffer_length;
 static double opt_max_statement_time= 0.0;
 static MYSQL *mysql=0;
-static DYNAMIC_STRING insert_pat, select_field_names, select_field_names_for_header;
+static DYNAMIC_STRING insert_pat, select_field_names,
+                      select_field_names_for_header, insert_field_names;
 static char  *opt_password=0,*current_user=0,
              *current_host=0,*path=0,*fields_terminated=0,
              *lines_terminated=0, *enclosed=0, *opt_enclosed=0, *escaped=0,
@@ -364,8 +365,15 @@ static struct my_option my_long_options[] =
   {"dump-date", 0, "Put a dump date to the end of the output.",
    &opt_dump_date, &opt_dump_date, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0},
   {"dump-history", 'H', "Dump system-versioned tables with history (only for "
-    "timestamp based versioning)", &opt_dump_history,
-    &opt_dump_history, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+    "timestamp based versioning). Use also --update-history if "
+   "upgrading to MariaDB 11.5 or newer from a version before 11.5",
+   &opt_dump_history, &opt_dump_history, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+  {"update-history", OPT_UPDATE_HISTORY,
+   "Update row_end history timestamp to support dates up to year 2106. "
+   "This option will also enable tz-utc. "
+   "Should be used when upgrading to MariaDB 11.5 or above.",
+   &opt_update_history, &opt_update_history,
+   0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"dump-slave", OPT_MYSQLDUMP_SLAVE_DATA,
    "This causes the binary log position and filename of the master to be "
    "appended to the dumped data output. Setting the value to 1, will print"
@@ -1345,23 +1353,31 @@ static int get_options(int *argc, char ***argv)
     fprintf(stderr, "%s: --xml can't be used with --tab or --dir.\n", my_progname_short);
     return(EX_USAGE);
   }
-  if (opt_xml && opt_dump_history)
-  {
-    fprintf(stderr, "%s: --xml can't be used with --dump-history.\n",
-            my_progname_short);
-    return(EX_USAGE);
-  }
-  if (opt_replace_into && opt_dump_history)
-  {
-    fprintf(stderr, "%s: --dump-history can't be used with --replace.\n",
-            my_progname_short);
-    return(EX_USAGE);
-  }
-  if (opt_asof_timestamp && opt_dump_history)
+  if (opt_dump_history)
   {
-    fprintf(stderr, "%s: --dump-history can't be used with --as-of.\n",
-            my_progname_short);
-    return(EX_USAGE);
+    if (opt_update_history)
+    {
+      /* dump history requires timezone "+00:00" */
+      opt_tz_utc= 1;
+    }
+    if (opt_xml)
+    {
+      fprintf(stderr, "%s: --xml can't be used with --dump-history.\n",
+              my_progname_short);
+      return(EX_USAGE);
+    }
+    if (opt_replace_into)
+    {
+      fprintf(stderr, "%s: --dump-history can't be used with --replace.\n",
+              my_progname_short);
+      return(EX_USAGE);
+    }
+    if (opt_asof_timestamp)
+    {
+      fprintf(stderr, "%s: --dump-history can't be used with --as-of.\n",
+              my_progname_short);
+      return(EX_USAGE);
+    }
   }
   if (opt_asof_timestamp && strchr(opt_asof_timestamp, '\''))
   {
@@ -1997,6 +2013,7 @@ static void free_resources()
   dynstr_free(&insert_pat);
   dynstr_free(&select_field_names);
   dynstr_free(&select_field_names_for_header);
+  dynstr_free(&insert_field_names);
   if (defaults_argv)
     free_defaults(defaults_argv);
   mysql_library_end();
@@ -3122,12 +3139,13 @@ static uint get_table_structure(const char *table, const char *db, char *table_t
   my_ulonglong num_fields;
   char       *result_table, *opt_quoted_table;
   const char *insert_option;
-  char	     name_buff[NAME_LEN+3],table_buff[NAME_LEN*2+3];
+  char	     name_buff[NAME_LEN*2+3],table_buff[NAME_LEN*2+3];
   char       table_buff2[NAME_LEN*2+3], query_buff[QUERY_LENGTH];
   char       temp_buff[NAME_LEN*2 + 3], temp_buff2[NAME_LEN*2 + 3];
+  char       *last_name;
   FILE       *sql_file= md_result_file;
   size_t     len;
-  my_bool    is_log_table;
+  my_bool    is_log_table, dummy_versioned;
   MYSQL_RES  *result;
   MYSQL_ROW  row;
   const char *s3_engine_ptr;
@@ -3166,12 +3184,14 @@ static uint get_table_structure(const char *table, const char *db, char *table_t
   {
     select_field_names_inited= 1;
     init_dynamic_string_checked(&select_field_names, "", 1024, 1024);
+    init_dynamic_string_checked(&insert_field_names, "", 1024, 1024);
     if (opt_header)
       init_dynamic_string_checked(&select_field_names_for_header, "", 1024, 1024);
   }
   else
   {
     dynstr_set_checked(&select_field_names, "");
+    dynstr_set_checked(&insert_field_names, "");
     if (opt_header)
       dynstr_set_checked(&select_field_names_for_header, "");
   }
@@ -3184,7 +3204,7 @@ static uint get_table_structure(const char *table, const char *db, char *table_t
   if (versioned)
   {
     if (!opt_asof_timestamp && !opt_dump_history)
-      versioned= NULL;
+      *versioned= 0;
     else
     {
       my_snprintf(query_buff, sizeof(query_buff), "select 1 from"
@@ -3200,6 +3220,11 @@ static uint get_table_structure(const char *table, const char *db, char *table_t
         *versioned= 0;
     }
   }
+  else
+  {
+    versioned= &dummy_versioned;
+    dummy_versioned= 0;
+  }
 
   len= my_snprintf(query_buff, sizeof(query_buff),
                    "SET SQL_QUOTE_SHOW_CREATE=%d", opt_quoted || opt_keywords);
@@ -3215,7 +3240,7 @@ static uint get_table_structure(const char *table, const char *db, char *table_t
 
   if (!opt_xml && !mysql_query_with_error_report(mysql, 0, query_buff))
   {
-    int vers_hidden= opt_dump_history && versioned && *versioned;
+    int vers_hidden= opt_dump_history && *versioned;
     /* using SHOW CREATE statement */
     if (!opt_no_create_info)
     {
@@ -3442,12 +3467,28 @@ static uint get_table_structure(const char *table, const char *db, char *table_t
       if (init)
       {
         dynstr_append_checked(&select_field_names, ", ");
+        dynstr_append_checked(&insert_field_names, ", ");
         if (opt_header)
           dynstr_append_checked(&select_field_names_for_header, ", ");
       }
       init=1;
-      dynstr_append_checked(&select_field_names,
-                            quote_name(row[0], name_buff, 0));
+
+      last_name= quote_name(row[0], name_buff, 0);
+      if (opt_dump_history && *versioned && opt_update_history &&
+          row[2] && strcmp(row[2], "ROW END") == 0)
+      {
+        dynstr_append_checked(&select_field_names, "if(");
+        dynstr_append_checked(&select_field_names, last_name);
+        dynstr_append_checked(&select_field_names,
+                              "= \"2038-01-19 03:14:07.999999\","
+                              "\"2106-02-07 06:28:15.999999\", ");
+        dynstr_append_checked(&select_field_names, last_name);
+        dynstr_append_checked(&select_field_names, ") as ");
+        dynstr_append_checked(&select_field_names, last_name);
+      }
+      else
+        dynstr_append_checked(&select_field_names, last_name);
+      dynstr_append_checked(&insert_field_names, last_name);
       if (opt_header)
         dynstr_append_checked(&select_field_names_for_header,
                               quote_for_equal(row[0], name_buff));
@@ -3456,7 +3497,14 @@ static uint get_table_structure(const char *table, const char *db, char *table_t
     if (vers_hidden)
     {
       complete_insert= 1;
-      dynstr_append_checked(&select_field_names, ", row_start, row_end");
+      dynstr_append_checked(&select_field_names, ", row_start,");
+      dynstr_append_checked(&select_field_names,
+                            opt_update_history ?
+                            "if(row_end = \"2038-01-19 03:14:07.999999\","
+                            "\"2106-02-07 06:28:15.999999\", row_end) as "
+                            "row_end" :
+                            "row_end");
+      dynstr_append_checked(&insert_field_names, ", row_start, row_end");
     }
 
     /*
@@ -3488,7 +3536,7 @@ static uint get_table_structure(const char *table, const char *db, char *table_t
     }
 
     if (complete_insert)
-      dynstr_append_checked(&insert_pat, select_field_names.str);
+      dynstr_append_checked(&insert_pat, insert_field_names.str);
     num_fields= mysql_num_rows(result) + (vers_hidden ? 2 : 0);
     mysql_free_result(result);
   }
@@ -3751,6 +3799,7 @@ static uint get_table_structure(const char *table, const char *db, char *table_t
   DBUG_RETURN((uint) num_fields);
 } /* get_table_structure */
 
+
 static void dump_trigger_old(FILE *sql_file, MYSQL_RES *show_triggers_rs,
                              MYSQL_ROW *show_trigger_row,
                              const char *table_name)