Merge branch '10.2-backup-fixes' into 10.2

appveyor.yml

@@ -3,7 +3,7 @@ version: build-{build}~branch-{branch}
 before_build:
   - md %APPVEYOR_BUILD_FOLDER%\win_build
   - cd %APPVEYOR_BUILD_FOLDER%\win_build
-  - cmake ..  -G "Visual Studio 15 2017 Win64" -DWITH_UNIT_TESTS=0 -DBISON_EXECUTABLE=C:\cygwin\bin\bison
+  - cmake ..  -G "Visual Studio 15 2017 Win64" -DWITH_UNIT_TESTS=0
 
 build:
   project: win_build\MySQL.sln

cmake/bison.cmake

@@ -20,7 +20,16 @@ IF(CMAKE_SYSTEM_NAME MATCHES "SunOS")
    SET(BISON_EXECUTABLE /opt/csw/bin/bison)
  ENDIF()
 ENDIF()
-FIND_PROGRAM(BISON_EXECUTABLE bison DOC "path to the bison executable")
+IF(WIN32)
+  SET(BISON_PATH_HINTS
+    HINTS
+    C:/gnuwin32/bin
+    C:/cygwin64/bin
+    C:/cygwin/bin)
+ENDIF()
+FIND_PROGRAM(BISON_EXECUTABLE bison
+  ${BISON_PATH_HINTS}
+  DOC "path to the bison executable")
 MARK_AS_ADVANCED(BISON_EXECUTABLE "")
 IF(NOT BISON_EXECUTABLE)
   MESSAGE("Warning: Bison executable not found in PATH")

cmake/os/Windows.cmake

@@ -24,9 +24,6 @@ INCLUDE (CheckCSourceRuns)
 INCLUDE (CheckSymbolExists)
 INCLUDE (CheckTypeSize)
 
-# Optionally read user configuration, generated by configure.js.
-# This is left for backward compatibility reasons only.
-INCLUDE(${CMAKE_BINARY_DIR}/win/configure.data OPTIONAL)
 
 # avoid running system checks by using pre-cached check results
 # system checks are expensive on VS since every tiny program is to be compiled in 
@@ -63,6 +60,8 @@ IF(MINGW AND CMAKE_SIZEOF_VOID_P EQUAL 4)
 ENDIF()
 
 IF(MSVC)
+  # Disable mingw based pkg-config found in Strawberry perl
+  SET(PKG_CONFIG_EXECUTABLE 0 CACHE INTERNAL "")
   SET(MSVC_CRT_TYPE /MT CACHE STRING
     "Runtime library - specify runtime library for linking (/MT,/MTd,/MD,/MDd)"
   )
@@ -156,7 +155,11 @@ IF(MSVC)
     SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /WX")
     SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /WX")
   ENDIF()
-
+  IF(MSVC_VERSION LESS 1910)
+    # Noisy warning C4800: 'type': forcing value to bool 'true' or 'false' (performance warning),
+    # removed in VS2017
+    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4800")
+  ENDIF()
 ENDIF()
 
 # Always link with socket library

extra/mariabackup/backup_copy.cc

@@ -1464,12 +1464,10 @@ bool backup_finish()
 		return(false);
 	}
 
-	if (!write_xtrabackup_info(mysql_connection)) {
+	if (!write_xtrabackup_info(mysql_connection, XTRABACKUP_INFO, opt_history != 0)) {
 		return(false);
 	}
 
-
-
 	return(true);
 }
 

extra/mariabackup/backup_mysql.cc

@@ -1398,7 +1398,7 @@ PERCONA_SCHEMA.xtrabackup_history and writes a new history record to the
 table containing all the history info particular to the just completed
 backup. */
 bool
-write_xtrabackup_info(MYSQL *connection)
+write_xtrabackup_info(MYSQL *connection, const char * filename, bool history)
 {
 
 	char *uuid = NULL;
@@ -1426,7 +1426,7 @@ write_xtrabackup_info(MYSQL *connection)
 		|| xtrabackup_databases_exclude
 		);
 
-	backup_file_printf(XTRABACKUP_INFO,
+	backup_file_printf(filename,
 		"uuid = %s\n"
 		"name = %s\n"
 		"tool_name = %s\n"
@@ -1463,7 +1463,7 @@ write_xtrabackup_info(MYSQL *connection)
 		xb_stream_name[xtrabackup_stream_fmt], /* format */
 		xtrabackup_compress ? "compressed" : "N"); /* compressed */
 
-	if (!opt_history) {
+	if (!history) {
 		goto cleanup;
 	}
 

extra/mariabackup/backup_mysql.h

@@ -68,7 +68,7 @@ bool
 write_binlog_info(MYSQL *connection);
 
 bool
-write_xtrabackup_info(MYSQL *connection);
+write_xtrabackup_info(MYSQL *connection, const char * filename, bool history);
 
 bool
 write_backup_config_file();

extra/mariabackup/xtrabackup.cc

@@ -3513,7 +3513,13 @@ xtrabackup_backup_low()
 			    "to '%s'.\n", filename);
 			return false;
 		}
-
+		sprintf(filename, "%s/%s", xtrabackup_extra_lsndir,
+			XTRABACKUP_INFO);
+		if (!write_xtrabackup_info(mysql_connection, filename, false)) {
+			msg("mariabackup: Error: failed to write info "
+			 "to '%s'.\n", filename);
+			return false;
+		}
 	}
 
 	return true;
@@ -3569,7 +3575,9 @@ xtrabackup_backup_func()
         if(innodb_init_param()) {
 fail:
 		stop_backup_threads();
-		innodb_shutdown();
+		if (fil_system) {
+			innodb_shutdown();
+		}
 		return(false);
 	}
 

mysql-test/collections/buildbot_suites.bat

@@ -1,4 +1,5 @@
 perl mysql-test-run.pl --verbose-restart --force --suite-timeout=120 --max-test-fail=10 --retry=3  --parallel=4 --suite=^
+vcol,gcol,perfschema,^
 main,^
 innodb,^
 plugins,^

mysql-test/mysql-test-run.pl

@@ -431,16 +431,21 @@ sub main {
   my $num_tests= @$tests;
   if ( $opt_parallel eq "auto" ) {
     # Try to find a suitable value for number of workers
-    my $sys_info= My::SysInfo->new();
-
-    $opt_parallel= $sys_info->num_cpus();
-    for my $limit (2000, 1500, 1000, 500){
-      $opt_parallel-- if ($sys_info->min_bogomips() < $limit);
+    if (IS_WINDOWS)
+    {
+      $opt_parallel= $ENV{NUMBER_OF_PROCESSORS} || 1;
+    }
+    else
+    {
+      my $sys_info= My::SysInfo->new();
+      $opt_parallel= $sys_info->num_cpus();
+      for my $limit (2000, 1500, 1000, 500){
+        $opt_parallel-- if ($sys_info->min_bogomips() < $limit);
+      }
     }
     my $max_par= $ENV{MTR_MAX_PARALLEL} || 8;
     $opt_parallel= $max_par if ($opt_parallel > $max_par);
     $opt_parallel= $num_tests if ($opt_parallel > $num_tests);
-    $opt_parallel= 1 if (IS_WINDOWS and $sys_info->isvm());
     $opt_parallel= 1 if ($opt_parallel < 1);
     mtr_report("Using parallel: $opt_parallel");
   }

mysql-test/suite/mariabackup/extra_lsndir.result

@@ -0,0 +1,2 @@
+xtrabackup_checkpoints
+xtrabackup_info

mysql-test/suite/mariabackup/extra_lsndir.test

@@ -0,0 +1,9 @@
+let $targetdir=$MYSQLTEST_VARDIR/tmp/backup;
+let $extra_lsndir=$MYSQLTEST_VARDIR/tmp/extra_lsndir;
+mkdir $extra_lsndir;
+--disable_result_log
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --extra-lsndir=$extra_lsndir;
+--enable_result_log
+list_files $extra_lsndir;
+rmdir $extra_lsndir;
+rmdir $targetdir;

mysql-test/suite/unit/suite.pm

@@ -40,8 +40,8 @@ sub start_test {
   my $bin=$ENV{MTR_BINDIR} || '..';
   return "Not run for embedded server" if $::opt_embedded_server;
   return "Not configured to run ctest" unless -f "$bin/CTestTestfile.cmake";
-  my ($ctest_vs)= $opt_vs_config ? "--build-config $opt_vs_config" : "";
-  my (@ctest_list)= `cd "$bin" && ctest $opt_vs_config --show-only --verbose`;
+  my ($ctest_vs)= $opt_vs_config ? "-C $opt_vs_config" : "";
+  my (@ctest_list)= `cd "$bin" && ctest $ctest_vs --show-only --verbose`;
   return "No ctest" if $?;
 
   my ($command, %tests, $prefix);
@@ -51,7 +51,9 @@ sub start_test {
       $command= $';
       $prefix= /libmariadb/ ? 'conc_' : '';
     } elsif (/^ +Test +#\d+: +/) {
-      $tests{$prefix.$'}=$command;
+      if ($command ne "NOT_AVAILABLE") {
+        $tests{$prefix.$'}=$command;
+      }
     }
   }
   bless { ctests => { %tests } };

storage/innobase/handler/ha_innodb.cc

@@ -15046,7 +15046,7 @@ ha_innobase::check(
 		    && !dict_index_is_corrupted(index)) {
 			/* Enlarge the fatal lock wait timeout during
 			CHECK TABLE. */
-			my_atomic_addlint(
+			my_atomic_addlong(
 				&srv_fatal_semaphore_wait_threshold,
 				SRV_SEMAPHORE_WAIT_EXTENSION);
 
@@ -15055,7 +15055,7 @@ ha_innobase::check(
 
 			/* Restore the fatal lock wait timeout after
 			CHECK TABLE. */
-			my_atomic_addlint(
+			my_atomic_addlong(
 				&srv_fatal_semaphore_wait_threshold,
 				-SRV_SEMAPHORE_WAIT_EXTENSION);
 

storage/innobase/include/buf0buf.h

@@ -1827,7 +1827,7 @@ struct buf_block_t{
 } while (0)
 #  define assert_block_ahi_valid(block)					\
 	ut_a((block)->index						\
-	     || my_atomic_addlint(&(block)->n_pointers, 0) == 0)
+	     || my_atomic_loadlint(&(block)->n_pointers) == 0)
 # else /* UNIV_AHI_DEBUG || UNIV_DEBUG */
 #  define assert_block_ahi_empty(block) /* nothing */
 #  define assert_block_ahi_empty_on_init(block) /* nothing */

storage/innobase/include/sync0types.h

@@ -1156,9 +1156,41 @@ enum rw_lock_flag_t {
 #endif /* UNIV_INNOCHECKSUM */
 
 #ifdef _WIN64
-#define my_atomic_addlint(A,B) my_atomic_add64((int64*) (A), (B))
-#define my_atomic_loadlint(A) my_atomic_load64((int64*) (A))
-#define my_atomic_caslint(A,B,C) my_atomic_cas64((int64*) (A), (int64*) (B), (C))
+static inline ulint my_atomic_addlint(ulint *A, ulint B)
+{
+  return ulint(my_atomic_add64((volatile int64*)A, B));
+}
+
+static inline ulint my_atomic_loadlint(const ulint *A)
+{
+  return ulint(my_atomic_load64((volatile int64*)A));
+}
+
+static inline lint my_atomic_addlint(volatile lint *A, lint B)
+{
+  return my_atomic_add64((volatile int64*)A, B);
+}
+
+static inline lint my_atomic_loadlint(const lint *A)
+{
+  return lint(my_atomic_load64((volatile int64*)A));
+}
+
+static inline void my_atomic_storelint(ulint *A, ulint B)
+{
+  my_atomic_store64((volatile int64*)A, B);
+}
+
+static inline lint my_atomic_caslint(volatile lint *A, lint *B, lint C)
+{
+  return my_atomic_cas64((volatile int64*)A, (int64 *)B, C);
+}
+
+static inline ulint my_atomic_caslint(ulint *A, ulint *B, ulint C)
+{
+  return my_atomic_cas64((volatile int64*)A, (int64 *)B, (int64)C);
+}
+
 #else
 #define my_atomic_addlint my_atomic_addlong
 #define my_atomic_loadlint my_atomic_loadlong
@@ -1188,7 +1220,7 @@ struct MY_ALIGNED(CPU_LEVEL1_DCACHE_LINESIZE) simple_counter
 #pragma warning (push)
 #pragma warning (disable : 4244)
 #endif
-			return Type(my_atomic_addlint(reinterpret_cast<lint*>
+			return Type(my_atomic_addlint(reinterpret_cast<ulint*>
 						      (&m_counter), i));
 #ifdef _MSC_VER
 #pragma warning (pop)

storage/innobase/include/trx0purge.h

@@ -514,9 +514,9 @@ class purge_sys_t
 					parallelized purge operation */
 	ReadView	view;		/*!< The purge will not remove undo logs
 					which are >= this view (purge view) */
-	volatile ulint	n_submitted;	/*!< Count of total tasks submitted
+	ulint	n_submitted;	/*!< Count of total tasks submitted
 					to the task queue */
-	volatile ulint	n_completed;	/*!< Count of total tasks completed */
+	ulint	n_completed;	/*!< Count of total tasks completed */
 
 	/*------------------------------*/
 	/* The following two fields form the 'purge pointer' which advances

storage/innobase/srv/srv0conc.cc

@@ -78,11 +78,11 @@ struct srv_conc_t {
 	This is no longer true. We'll, however, keep the lint datatype to add
 	assertions to catch any corner cases that we may have missed. */
 
-	volatile lint	n_active;
+	ulint	n_active;
 
 	/** Number of OS threads waiting in the FIFO for permission to
 	enter InnoDB */
-	volatile lint	n_waiting;
+	ulint	n_waiting;
 };
 
 /* Control variables for tracking concurrency. */
@@ -152,7 +152,7 @@ srv_conc_enter_innodb_with_atomics(
 			return;
 		}
 
-		if (srv_conc.n_active < (lint) srv_thread_concurrency) {
+		if (srv_conc.n_active < srv_thread_concurrency) {
 			ulint	n_active;
 
 			/* Check if there are any free tickets. */

storage/innobase/srv/srv0srv.cc

@@ -2588,8 +2588,8 @@ DECLARE_THREAD(srv_worker_thread)(
 	slot = srv_reserve_slot(SRV_WORKER);
 
 	ut_a(srv_n_purge_threads > 1);
-	ut_a(my_atomic_loadlint(&srv_sys.n_threads_active[SRV_WORKER])
-	     < static_cast<lint>(srv_n_purge_threads));
+	ut_a(ulong(my_atomic_loadlint(&srv_sys.n_threads_active[SRV_WORKER]))
+	     < srv_n_purge_threads);
 
 	/* We need to ensure that the worker threads exit after the
 	purge coordinator thread. Otherwise the purge coordinator can