Merge 10.3 into 10.4

We will expose some more std::atomic internals in Atomic_counter,
so that dict_index_t::lock will support the default assignment operator.

client/mysqltest.cc

@@ -587,9 +587,10 @@ ATTRIBUTE_NORETURN
 static void cleanup_and_exit(int exit_code);
 
 ATTRIBUTE_NORETURN
-void really_die(const char *msg);
+static void really_die(const char *msg);
 void report_or_die(const char *fmt, ...);
-void die(const char *fmt, ...);
+ATTRIBUTE_NORETURN
+static void die(const char *fmt, ...);
 static void make_error_message(char *buf, size_t len, const char *fmt, va_list args);
 ATTRIBUTE_NORETURN ATTRIBUTE_FORMAT(printf, 1, 2)
 void abort_not_supported_test(const char *fmt, ...);
@@ -1540,7 +1541,7 @@ static void make_error_message(char *buf, size_t len, const char *fmt, va_list a
   s+= my_snprintf(s, end -s, "\n");
 }
 
-void die(const char *fmt, ...)
+static void die(const char *fmt, ...)
 {
   char buff[DIE_BUFF_SIZE];
   va_list args;
@@ -1549,7 +1550,7 @@ void die(const char *fmt, ...)
   really_die(buff);
 }
 
-void really_die(const char *msg)
+static void really_die(const char *msg)
 {
   static int dying= 0;
   fflush(stdout);

include/my_counter.h

@@ -1,7 +1,7 @@
 #ifndef MY_COUNTER_H_INCLUDED
 #define MY_COUNTER_H_INCLUDED
 /*
-   Copyright (C) 2018 MariaDB Foundation
+   Copyright (C) 2018, 2020, MariaDB
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -45,5 +45,16 @@ template <typename Type> class Atomic_counter
   operator Type() const { return m_counter.load(std::memory_order_relaxed); }
   Type operator=(const Type val)
   { m_counter.store(val, std::memory_order_relaxed); return val; }
+  Type operator=(const Atomic_counter<Type> &rhs) { return *this= Type{rhs}; }
+
+  Type fetch_add(const Type i, std::memory_order m)
+  { return m_counter.fetch_add(i, m); }
+  Type fetch_sub(const Type i, std::memory_order m)
+  { return m_counter.fetch_sub(i, m); }
+  bool compare_exchange_strong(Type& i1, const Type i2,
+                               std::memory_order m1, std::memory_order m2)
+  { return m_counter.compare_exchange_strong(i1, i2, m1, m2); }
+  Type exchange(const Type i, std::memory_order m)
+  { return m_counter.exchange(i, m); }
 };
 #endif /* MY_COUNTER_H_INCLUDED */

include/my_valgrind.h

@@ -1,4 +1,4 @@
-/* Copyright (C) 2010, 2019, MariaDB Corporation.
+/* Copyright (C) 2010, 2020, MariaDB Corporation.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -32,7 +32,9 @@
 
 #if defined(HAVE_VALGRIND_MEMCHECK_H) && defined(HAVE_valgrind)
 # include <valgrind/memcheck.h>
+# define HAVE_valgrind_or_MSAN
 # define MEM_UNDEFINED(a,len) VALGRIND_MAKE_MEM_UNDEFINED(a,len)
+# define MEM_MAKE_DEFINED(a,len) VALGRIND_MAKE_MEM_DEFINED(a,len)
 # define MEM_NOACCESS(a,len) VALGRIND_MAKE_MEM_NOACCESS(a,len)
 # define MEM_CHECK_ADDRESSABLE(a,len) VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a,len)
 # define MEM_CHECK_DEFINED(a,len) VALGRIND_CHECK_MEM_IS_DEFINED(a,len)
@@ -42,28 +44,42 @@
 /* How to do manual poisoning:
 https://github.com/google/sanitizers/wiki/AddressSanitizerManualPoisoning */
 # define MEM_UNDEFINED(a,len) ASAN_UNPOISON_MEMORY_REGION(a,len)
+# define MEM_MAKE_DEFINED(a,len) ((void) 0)
 # define MEM_NOACCESS(a,len) ASAN_POISON_MEMORY_REGION(a,len)
 # define MEM_CHECK_ADDRESSABLE(a,len) ((void) 0)
 # define MEM_CHECK_DEFINED(a,len) ((void) 0)
 # define REDZONE_SIZE 8
+#elif __has_feature(memory_sanitizer)
+# include <sanitizer/msan_interface.h>
+# define HAVE_valgrind_or_MSAN
+# define MEM_UNDEFINED(a,len) __msan_allocated_memory(a,len)
+# define MEM_MAKE_DEFINED(a,len) __msan_unpoison(a,len)
+# define MEM_NOACCESS(a,len) ((void) 0)
+# define MEM_CHECK_ADDRESSABLE(a,len) ((void) 0)
+# define MEM_CHECK_DEFINED(a,len) __msan_check_mem_is_initialized(a,len)
+# define REDZONE_SIZE 8
 #else
 # define MEM_UNDEFINED(a,len) ((void) (a), (void) (len))
+# define MEM_MAKE_DEFINED(a,len) ((void) 0)
 # define MEM_NOACCESS(a,len) ((void) 0)
 # define MEM_CHECK_ADDRESSABLE(a,len) ((void) 0)
 # define MEM_CHECK_DEFINED(a,len) ((void) 0)
 # define REDZONE_SIZE 0
 #endif /* HAVE_VALGRIND_MEMCHECK_H */
 
-#if defined(TRASH_FREED_MEMORY)
-/* NOTE: Do not invoke TRASH_FILL directly! Use TRASH_ALLOC or TRASH_FREE.
-
-The MEM_UNDEFINED() call before memset() is for canceling the effect
-of any previous MEM_NOACCESS(). We must invoke MEM_UNDEFINED() after
-writing the dummy pattern, unless MEM_NOACCESS() is going to be invoked.
-On AddressSanitizer, the MEM_UNDEFINED() in TRASH_ALLOC() has no effect. */
+#ifdef TRASH_FREED_MEMORY
+/*
+  TRASH_FILL() has to call MEM_UNDEFINED() to cancel any effect of TRASH_FREE().
+  This can happen in the case one does
+  TRASH_ALLOC(A,B) ; TRASH_FREE(A,B) ; TRASH_ALLOC(A,B)
+  to reuse the same memory in an internal memory allocator like MEM_ROOT.
+  For my_malloc() and safemalloc() the extra MEM_UNDEFINED is bit of an
+  overkill.
+  TRASH_FILL() is an internal function and should not be used externally.
+*/
 #define TRASH_FILL(A,B,C) do { const size_t trash_tmp= (B); MEM_UNDEFINED(A, trash_tmp); memset(A, C, trash_tmp); } while (0)
 #else
-#define TRASH_FILL(A,B,C) while (0)
+#define TRASH_FILL(A,B,C) do { MEM_UNDEFINED((A), (B)); } while (0)
 #endif
 /** Note that some memory became allocated or uninitialized. */
 #define TRASH_ALLOC(A,B) do { TRASH_FILL(A,B,0xA5); MEM_UNDEFINED(A,B); } while(0)

include/span.h

@@ -24,11 +24,33 @@ this program; if not, write to the Free Software Foundation, Inc.,
 namespace st_
 {
 
+namespace detail
+{
+
+template <class T> struct remove_cv
+{
+  typedef T type;
+};
+template <class T> struct remove_cv<const T>
+{
+  typedef T type;
+};
+template <class T> struct remove_cv<volatile T>
+{
+  typedef T type;
+};
+template <class T> struct remove_cv<const volatile T>
+{
+  typedef T type;
+};
+
+} // namespace detail
+
 template <class ElementType> class span
 {
 public:
   typedef ElementType element_type;
-  typedef ElementType value_type;
+  typedef typename detail::remove_cv<ElementType>::type value_type;
   typedef size_t size_type;
   typedef ptrdiff_t difference_type;
   typedef element_type *pointer;
@@ -38,7 +60,6 @@ template <class ElementType> class span
   typedef pointer iterator;
   typedef const_pointer const_iterator;
   typedef std::reverse_iterator<iterator> reverse_iterator;
-  typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
 
   span() : data_(NULL), size_(0) {}
 
@@ -64,73 +85,72 @@ template <class ElementType> class span
 
   span &operator=(const span &other)
   {
-    data_= other.data_;
-    size_= other.size_;
+    data_= other.data();
+    size_= other.size();
     return *this;
   }
 
   template <size_t Count> span<element_type> first() const
   {
     assert(!empty());
-    return span(data_, 1);
+    return span(data(), 1);
   }
   template <size_t Count> span<element_type> last() const
   {
     assert(!empty());
-    return span(data_ + size() - 1, 1);
+    return span(data() + size() - 1, 1);
   }
 
   span<element_type> first(size_type count) const
   {
     assert(!empty());
-    return span(data_, 1);
+    return span(data(), 1);
   }
   span<element_type> last(size_type count) const
   {
     assert(!empty());
-    return span(data_ + size() - 1, 1);
+    return span(data() + size() - 1, 1);
   }
   span<element_type> subspan(size_type offset, size_type count) const
   {
     assert(!empty());
     assert(size() >= offset + count);
-    return span(data_ + offset, count);
+    return span(data() + offset, count);
   }
 
   size_type size() const { return size_; }
-  size_type size_bytes() const { return size_ * sizeof(ElementType); }
-  bool empty() const __attribute__((warn_unused_result)) { return size_ == 0; }
+  size_type size_bytes() const { return size() * sizeof(ElementType); }
+  bool empty() const __attribute__((warn_unused_result))
+  {
+    return size() == 0;
+  }
 
   reference operator[](size_type idx) const
   {
     assert(size() > idx);
-    return data_[idx];
+    return data()[idx];
   }
   reference front() const
   {
     assert(!empty());
-    return data_[0];
+    return data()[0];
   }
   reference back() const
   {
     assert(!empty());
-    return data_[size() - 1];
-  }
-  pointer data() const
-  {
-    assert(!empty());
-    return data_;
+    return data()[size() - 1];
   }
+  pointer data() const { return data_; }
 
   iterator begin() const { return data_; }
   iterator end() const { return data_ + size_; }
   reverse_iterator rbegin() const
   {
-    return std::reverse_iterator<iterator>(std::advance(end(), -1));
+    return std::reverse_iterator<iterator>(end());
   }
   reverse_iterator rend() const
   {
-    return std::reverse_iterator<iterator>(std::advance(begin(), -1));
+    return std::reverse_iterator<iterator>(begin());
   }
 
 private:

mysql-test/suite/encryption/t/innodb_encryption_tables.test

@@ -1,6 +1,8 @@
 -- source include/have_innodb.inc
 -- source include/have_example_key_management_plugin.inc
 -- source include/not_embedded.inc
+# We can't run this test under valgrind as it 'takes forever'
+-- source include/not_valgrind.inc
 
 create table innodb_normal(c1 bigint not null, b char(200)) engine=innodb;
 create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact;

mysql-test/suite/gcol/r/innodb_virtual_debug_purge.result

@@ -234,48 +234,3 @@ set global debug_dbug= @saved_dbug;
 drop table t1;
 set debug_sync=reset;
 SET GLOBAL innodb_purge_rseg_truncate_frequency = @saved_frequency;
-#
-# MDEV-18546 ASAN heap-use-after-free
-# in innobase_get_computed_value / row_purge
-#
-CREATE TABLE t1 (
-pk INT AUTO_INCREMENT,
-b BIT(15),
-v BIT(15) AS (b) VIRTUAL,
-PRIMARY KEY(pk),
-UNIQUE(v)
-) ENGINE=InnoDB;
-INSERT IGNORE INTO t1 (b) VALUES
-(NULL),(b'011'),(b'000110100'),
-(b'01101101010'),(b'01111001001011'),(NULL);
-SET GLOBAL innodb_debug_sync = "ib_clust_v_col_before_row_allocated "
-                               "SIGNAL before_row_allocated "
-                               "WAIT_FOR flush_unlock";
-SET GLOBAL innodb_debug_sync = "ib_open_after_dict_open "
-                               "SIGNAL purge_open "
-                               "WAIT_FOR select_open";
-SET @saved_dbug= @@GLOBAL.debug_dbug;
-set global debug_dbug= "d,ib_purge_virtual_index_callback";
-connect purge_waiter,localhost,root;
-SET debug_sync= "now WAIT_FOR before_row_allocated";
-connection default;
-REPLACE INTO t1 (pk, b) SELECT pk, b FROM t1;
-connection purge_waiter;
-connection default;
-disconnect purge_waiter;
-FLUSH TABLES;
-SET GLOBAL innodb_debug_sync = reset;
-SET debug_sync= "now SIGNAL flush_unlock WAIT_FOR purge_open";
-SET GLOBAL innodb_debug_sync = reset;
-SET debug_sync= "ib_open_after_dict_open SIGNAL select_open";
-SELECT * FROM t1;
-pk	b	v
-1	NULL	NULL
-2		
-3	4	4
-4	j	j
-5	K	K
-6	NULL	NULL
-DROP TABLE t1;
-SET debug_sync= reset;
-set global debug_dbug= @saved_dbug;

mysql-test/suite/gcol/t/innodb_virtual_debug_purge.test

@@ -311,67 +311,3 @@ drop table t1;
 --source include/wait_until_count_sessions.inc
 set debug_sync=reset;
 SET GLOBAL innodb_purge_rseg_truncate_frequency = @saved_frequency;
-
---echo #
---echo # MDEV-18546 ASAN heap-use-after-free
---echo # in innobase_get_computed_value / row_purge
---echo #
-
-CREATE TABLE t1 (
-  pk INT AUTO_INCREMENT,
-  b BIT(15),
-  v BIT(15) AS (b) VIRTUAL,
-  PRIMARY KEY(pk),
-  UNIQUE(v)
-) ENGINE=InnoDB;
-INSERT IGNORE INTO t1 (b) VALUES
-  (NULL),(b'011'),(b'000110100'),
-  (b'01101101010'),(b'01111001001011'),(NULL);
-
-SET GLOBAL innodb_debug_sync = "ib_clust_v_col_before_row_allocated "
-                               "SIGNAL before_row_allocated "
-                               "WAIT_FOR flush_unlock";
-SET GLOBAL innodb_debug_sync = "ib_open_after_dict_open "
-                               "SIGNAL purge_open "
-                               "WAIT_FOR select_open";
-
-# In 10.2 trx_undo_roll_ptr_is_insert(t_roll_ptr) condition never pass in purge,
-# so this condition is forced to pass in row_vers_old_has_index_entry
-SET @saved_dbug= @@GLOBAL.debug_dbug;
-set global debug_dbug= "d,ib_purge_virtual_index_callback";
-
-# The purge starts from REPLACE command. To avoid possible race, separate
-# connection is used.
---connect(purge_waiter,localhost,root)
---send
-SET debug_sync= "now WAIT_FOR before_row_allocated";
-
---connection default
-REPLACE INTO t1 (pk, b) SELECT pk, b FROM t1;
-
---connection purge_waiter
-# Now we will definitely catch ib_clust_v_col_before_row_allocated
---reap
---connection default
---disconnect purge_waiter
-
-# purge hangs on the sync point. table is purged, ref_count is set to 0
-FLUSH TABLES;
-
-# Avoid hang on repeating purge.
-# Reset Will be applied after first record is purged
-SET GLOBAL innodb_debug_sync = reset;
-
-SET debug_sync= "now SIGNAL flush_unlock WAIT_FOR purge_open";
-
-# Avoid hang on repeating purge
-SET GLOBAL innodb_debug_sync = reset;
-
-# select unblocks purge thread
-SET debug_sync= "ib_open_after_dict_open SIGNAL select_open";
-SELECT * FROM t1;
-
-# Cleanup
-DROP TABLE t1;
-SET debug_sync= reset;
-set global debug_dbug= @saved_dbug;