BUG#19502202 - SERVER SHUTDOWN HANG SEEN IN SOME INNODB

               & RPL TESTS

A MTR test binlog.binlog_xa_prepared_disconnect fails sporadically
on PB2. When the server is shutdown during the execution of the tests,
the shutdown of the mysql server doesn't happen within a specified
timeout thereby causing SIGABRT to be sent to the server and the test
aborts.
As part of the shutdown sequence, sockets are shutdown and closed in
one thread and this shall cause the POLLHUP event to be generated for
the sockets that are stuck in poll sleep state on another thread. The
generation of POLLHUP from the kernel is async and not deterministic
and time-bound.  Hence shutdown hangs and could not complete in a
specified timeout period. The patch fixes this by using ppoll in
vio_io_wait and sending pthread_kill to the thread that is stuck
on poll when the socket is undergoing shutdown.
The constructor and destructor functionality to the patch was addded
by Tor Didriksen <tor.didriksen@oracle.com>.

Author: thayumanavar77

include/mysql.h.pp

@@ -83,10 +83,8 @@
   SERVER_STATUS_IN_TRANS_READONLY= 8192,
   SERVER_SESSION_STATE_CHANGED= (1UL << 14)
 };
-struct st_vio;
-typedef struct st_vio Vio;
 typedef struct st_net {
-  Vio *vio;
+  void *vio;
   unsigned char *buff,*buff_end,*write_pos,*read_pos;
   my_socket fd;
   unsigned long remain_in_buf,length, buf_length, where_b;
@@ -141,7 +139,7 @@
   SESSION_TRACK_TRANSACTION_CHARACTERISTICS,
   SESSION_TRACK_TRANSACTION_STATE
 };
-my_bool my_net_init(NET *net, Vio* vio);
+my_bool my_net_init(NET *net, void *vio);
 void my_net_local_init(NET *net);
 void net_end(NET *net);
 void net_clear(NET *net, my_bool check_buffer);

include/mysql_com.h

@@ -749,9 +749,13 @@ enum SERVER_STATUS_flags_enum
 */
 #define ONLY_KILL_QUERY         1
 
-
-struct st_vio;					/* Only C */
+#ifdef __cplusplus
+struct st_vio;
 typedef struct st_vio Vio;
+#define MYSQL_VIO Vio*
+#else
+#define MYSQL_VIO void*
+#endif
 
 #define MAX_TINYINT_WIDTH       3       /**< Max width for a TINY w.o. sign */
 #define MAX_SMALLINT_WIDTH      5       /**< Max width for a SHORT w.o. sign */
@@ -762,7 +766,7 @@ typedef struct st_vio Vio;
 #define MAX_BLOB_WIDTH		16777216	/**< Default width for blob */
 
 typedef struct st_net {
-  Vio *vio;
+  MYSQL_VIO vio;
   unsigned char *buff,*buff_end,*write_pos,*read_pos;
   my_socket fd;					/* For Perl DBI/dbd */
   /**
@@ -937,7 +941,7 @@ enum enum_session_state_type
 extern "C" {
 #endif
 
-my_bool	my_net_init(NET *net, Vio* vio);
+my_bool	my_net_init(NET *net, MYSQL_VIO vio);
 void my_net_local_init(NET *net);
 void net_end(NET *net);
 void net_clear(NET *net, my_bool check_buffer);

include/sql_common.h

@@ -189,7 +189,7 @@ void set_mysql_extended_error(MYSQL *mysql, int errcode, const char *sqlstate,
 
 /* client side of the pluggable authentication */
 struct st_plugin_vio_info;
-void mpvio_info(Vio *vio, struct st_plugin_vio_info *info);
+void mpvio_info(MYSQL_VIO vio, struct st_plugin_vio_info *info);
 int run_plugin_auth(MYSQL *mysql, char *data, uint data_len,
                     const char *data_plugin, const char *db);
 int mysql_client_plugin_init();

include/violite.h

@@ -28,6 +28,15 @@
 
 /* Simple vio interface in C;  The functions are implemented in violite.c */
 
+#if !defined(_WIN32) && !defined(__APPLE__) && !defined(__SUNPRO_CC)
+#define USE_PPOLL_IN_VIO
+#endif
+
+#if defined(__cplusplus) && defined(USE_PPOLL_IN_VIO)
+#include <signal.h>
+#include <atomic>
+#endif
+
 #ifdef	__cplusplus
 extern "C" {
 #endif /* __cplusplus */
@@ -38,6 +47,9 @@ void init_vio_psi_keys();
 
 #ifdef __cplusplus
 typedef struct st_vio Vio;
+#define MYSQL_VIO Vio*
+#else
+#define MYSQL_VIO void*
 #endif /* __cplusplus */
 
 enum enum_vio_type
@@ -104,13 +116,13 @@ enum enum_vio_io_event
 #define VIO_LOCALHOST 1                         /* a localhost connection */
 #define VIO_BUFFERED_READ 2                     /* use buffered read */
 #define VIO_READ_BUFFER_SIZE 16384              /* size of read buffer */
-#define VIO_DESCRIPTION_SIZE 30                 /* size of description */
 
-Vio* vio_new(my_socket sd, enum enum_vio_type type, uint flags);
-Vio*  mysql_socket_vio_new(MYSQL_SOCKET mysql_socket, enum enum_vio_type type, uint flags);
+MYSQL_VIO vio_new(my_socket sd, enum enum_vio_type type, uint flags);
+MYSQL_VIO  mysql_socket_vio_new(MYSQL_SOCKET mysql_socket, enum enum_vio_type type, uint flags);
+
 #ifdef _WIN32
-Vio* vio_new_win32pipe(HANDLE hPipe);
-Vio* vio_new_win32shared_memory(HANDLE handle_file_map,
+MYSQL_VIO vio_new_win32pipe(HANDLE hPipe);
+MYSQL_VIO vio_new_win32shared_memory(HANDLE handle_file_map,
                                 HANDLE handle_map,
                                 HANDLE event_server_wrote,
                                 HANDLE event_server_read,
@@ -121,41 +133,42 @@ Vio* vio_new_win32shared_memory(HANDLE handle_file_map,
 #define HANDLE void *
 #endif /* _WIN32 */
 
-void    vio_delete(Vio* vio);
-int vio_shutdown(Vio* vio);
-my_bool vio_reset(Vio* vio, enum enum_vio_type type,
+void    vio_delete(MYSQL_VIO vio);
+int vio_shutdown(MYSQL_VIO vio);
+my_bool vio_reset(MYSQL_VIO vio, enum enum_vio_type type,
                   my_socket sd, void *ssl, uint flags);
-size_t  vio_read(Vio *vio, uchar *	buf, size_t size);
-size_t  vio_read_buff(Vio *vio, uchar * buf, size_t size);
-size_t  vio_write(Vio *vio, const uchar * buf, size_t size);
+size_t  vio_read(MYSQL_VIO vio, uchar *	buf, size_t size);
+size_t  vio_read_buff(MYSQL_VIO vio, uchar * buf, size_t size);
+size_t  vio_write(MYSQL_VIO vio, const uchar * buf, size_t size);
 /* setsockopt TCP_NODELAY at IPPROTO_TCP level, when possible */
-int vio_fastsend(Vio *vio);
+int vio_fastsend(MYSQL_VIO vio);
 /* setsockopt SO_KEEPALIVE at SOL_SOCKET level, when possible */
-int vio_keepalive(Vio *vio, my_bool	onoff);
+int vio_keepalive(MYSQL_VIO vio, my_bool	onoff);
 /* Whenever we should retry the last read/write operation. */
-my_bool vio_should_retry(Vio *vio);
+my_bool vio_should_retry(MYSQL_VIO vio);
 /* Check that operation was timed out */
-my_bool vio_was_timeout(Vio *vio);
+my_bool vio_was_timeout(MYSQL_VIO vio);
 /* Short text description of the socket for those, who are curious.. */
-const char* vio_description(Vio *vio);
+#define VIO_DESCRIPTION_SIZE 30                 /* size of description */
+void vio_description(MYSQL_VIO vio, char *buf);
 /* Return the type of the connection */
-enum enum_vio_type vio_type(Vio* vio);
+enum enum_vio_type vio_type(MYSQL_VIO vio);
 /* Return last error number */
-int	vio_errno(Vio*vio);
+int	vio_errno(MYSQL_VIO vio);
 /* Get socket number */
-my_socket vio_fd(Vio*vio);
+my_socket vio_fd(MYSQL_VIO vio);
 /* Remote peer's address and name in text form */
-my_bool vio_peer_addr(Vio *vio, char *buf, uint16 *port, size_t buflen);
+my_bool vio_peer_addr(MYSQL_VIO vio, char *buf, uint16 *port, size_t buflen);
 /* Wait for an I/O event notification. */
-int vio_io_wait(Vio *vio, enum enum_vio_io_event event, int timeout);
-my_bool vio_is_connected(Vio *vio);
+int vio_io_wait(MYSQL_VIO vio, enum enum_vio_io_event event, int timeout);
+my_bool vio_is_connected(MYSQL_VIO vio);
 #ifndef DBUG_OFF
-ssize_t vio_pending(Vio *vio);
+ssize_t vio_pending(MYSQL_VIO vio);
 #endif
 /* Set timeout for a network operation. */
-int vio_timeout(Vio *vio, uint which, int timeout_sec);
+int vio_timeout(MYSQL_VIO vio, uint which, int timeout_sec);
 /* Connect to a peer. */
-my_bool vio_socket_connect(Vio *vio, struct sockaddr *addr, socklen_t len,
+my_bool vio_socket_connect(MYSQL_VIO vio, struct sockaddr *addr, socklen_t len,
                            int timeout);
 
 my_bool vio_get_normalized_ip_string(const struct sockaddr *addr, size_t addr_length,
@@ -207,8 +220,8 @@ struct st_VioSSLFd
   SSL_CTX *ssl_context;
 };
 
-int sslaccept(struct st_VioSSLFd*, Vio *, long timeout, unsigned long *errptr);
-int sslconnect(struct st_VioSSLFd*, Vio *, long timeout, unsigned long *errptr);
+int sslaccept(struct st_VioSSLFd*, MYSQL_VIO, long timeout, unsigned long *errptr);
+int sslconnect(struct st_VioSSLFd*, MYSQL_VIO, long timeout, unsigned long *errptr);
 
 struct st_VioSSLFd
 *new_VioSSLConnectorFd(const char *key_file, const char *cert_file,
@@ -258,32 +271,49 @@ enum SSL_type
   SSL_TYPE_SPECIFIED
 };
 
-
+#ifdef __cplusplus
 /* HFTODO - hide this if we don't want client in embedded server */
 /* This structure is for every connection on both sides */
 struct st_vio
 {
   MYSQL_SOCKET  mysql_socket;           /* Instrumented socket */
-  my_bool       localhost;              /* Are we from localhost? */
+  bool          localhost= { false };   /* Are we from localhost? */
+  enum_vio_type type= { NO_VIO_TYPE };  /* Type of connection */
+
+  int           read_timeout= { -1 };   /* Timeout value (ms) for read ops. */
+  int           write_timeout= { -1 };  /* Timeout value (ms) for write ops. */
+  int           retry_count= { 1 };     /* Retry count */
+  bool          inactive= { false };    /* Connection has been shutdown */
+
   struct sockaddr_storage   local;      /* Local internet address */
   struct sockaddr_storage   remote;     /* Remote internet address */
-  size_t addrLen;                       /* Length of remote address */
-  enum enum_vio_type    type;           /* Type of connection */
-  my_bool               inactive; /* Connection inactive (has been shutdown) */
-  char                  desc[VIO_DESCRIPTION_SIZE]; /* Description string. This
-                                                      member MUST NOT be
-                                                      used directly, but only
-                                                      via function
-                                                      "vio_description" */
-  char                  *read_buffer;   /* buffer for vio_read_buff */
-  char                  *read_pos;      /* start of unfetched data in the
-                                           read buffer */
-  char                  *read_end;      /* end of unfetched data */
-  int                   read_timeout;   /* Timeout value (ms) for read ops. */
-  int                   write_timeout;  /* Timeout value (ms) for write ops. */
-  int                   retry_count;    /* Retry count */
+  size_t        addrLen= { 0 };         /* Length of remote address */
+  char *read_buffer= { nullptr };   /* buffer for vio_read_buff */
+  char *read_pos= { nullptr };      /* start of unfetched data in the
+                                       read buffer */
+  char *read_end= { nullptr };      /* end of unfetched data */
+
+#ifdef USE_PPOLL_IN_VIO
+  my_thread_t           thread_id= { 0 }; // Thread PID
+  sigset_t              signal_mask;      // Signal mask
+  /*
+    Flag to indicate whether we are in poll or shutdown.
+    A true value of flag indicates either the socket
+    has called  shutdown or it is sleeping on a poll call.
+    False value of this flag means that the socket is
+    not sleeping on a poll call.
+    This flag provides synchronization between two threads
+    one entering vio_io_wait and another entering vio_shutdown
+    for the same socket. If the other thread is waiting on poll
+    sleep, it wakes up the thread by sending a signal via
+    pthread_kill. Also it ensures that no other thread enters in
+    to a poll call if it's socket has undergone shutdown.
 
-  /* 
+  */
+  std::atomic_flag      poll_shutdown_flag= ATOMIC_FLAG_INIT;
+#endif
+
+  /*
      VIO vtable interface to be implemented by VIO's like SSL, Socket,
      Named Pipe, etc.
   */
@@ -292,49 +322,61 @@ struct st_vio
      viodelete is responsible for cleaning up the VIO object by freeing 
      internal buffers, closing descriptors, handles. 
   */
-  void    (*viodelete)(Vio*);
-  int     (*vioerrno)(Vio*);
-  size_t  (*read)(Vio*, uchar *, size_t);
-  size_t  (*write)(Vio*, const uchar *, size_t);
-  int     (*timeout)(Vio*, uint, my_bool);
-  int     (*viokeepalive)(Vio*, my_bool);
-  int     (*fastsend)(Vio*);
-  my_bool (*peer_addr)(Vio*, char *, uint16*, size_t);
-  void    (*in_addr)(Vio*, struct sockaddr_storage*);
-  my_bool (*should_retry)(Vio*);
-  my_bool (*was_timeout)(Vio*);
+  void    (*viodelete)(MYSQL_VIO)= { nullptr };
+  int     (*vioerrno)(MYSQL_VIO)= { nullptr };
+  size_t  (*read)(MYSQL_VIO, uchar *, size_t)= { nullptr };
+  size_t  (*write)(MYSQL_VIO, const uchar *, size_t)= { nullptr };
+  int     (*timeout)(MYSQL_VIO, uint, my_bool)= { nullptr };
+  int     (*viokeepalive)(MYSQL_VIO, my_bool)= { nullptr };
+  int     (*fastsend)(MYSQL_VIO)= { nullptr };
+  my_bool (*peer_addr)(MYSQL_VIO, char *, uint16*, size_t)= { nullptr };
+  void    (*in_addr)(MYSQL_VIO, struct sockaddr_storage*)= { nullptr };
+  my_bool (*should_retry)(MYSQL_VIO)= { nullptr };
+  my_bool (*was_timeout)(MYSQL_VIO)= { nullptr };
   /* 
      vioshutdown is resposnible to shutdown/close the channel, so that no 
      further communications can take place, however any related buffers,
      descriptors, handles can remain valid after a shutdown.
   */
-  int     (*vioshutdown)(Vio*);
-  my_bool (*is_connected)(Vio*);
-  my_bool (*has_data) (Vio*);
-  int (*io_wait)(Vio*, enum enum_vio_io_event, int);
-  my_bool (*connect)(Vio*, struct sockaddr *, socklen_t, int);
+  int     (*vioshutdown)(MYSQL_VIO)= { nullptr };
+  my_bool (*is_connected)(MYSQL_VIO)= { nullptr };
+  my_bool (*has_data) (MYSQL_VIO)= { nullptr };
+  int (*io_wait)(MYSQL_VIO, enum enum_vio_io_event, int)= { nullptr };
+  my_bool (*connect)(MYSQL_VIO, struct sockaddr *, socklen_t, int)= { nullptr };
 #ifdef _WIN32
-  OVERLAPPED overlapped;
-  HANDLE hPipe;
+  OVERLAPPED overlapped = { 0 };
+  HANDLE hPipe { nullptr };
 #endif
 #ifdef HAVE_OPENSSL
-  void    *ssl_arg;
+  void    *ssl_arg= { nullptr };
 #endif
 #if defined (_WIN32) && !defined (EMBEDDED_LIBRARY)
-  HANDLE  handle_file_map;
-  char    *handle_map;
-  HANDLE  event_server_wrote;
-  HANDLE  event_server_read;
-  HANDLE  event_client_wrote;
-  HANDLE  event_client_read;
-  HANDLE  event_conn_closed;
-  size_t  shared_memory_remain;
-  char    *shared_memory_pos;
+  HANDLE  handle_file_map= { nullptr };
+  char    *handle_map= { nullptr };
+  HANDLE  event_server_wrote= { nullptr };
+  HANDLE  event_server_read= { nullptr };
+  HANDLE  event_client_wrote= { nullptr };
+  HANDLE  event_client_read= { nullptr };
+  HANDLE  event_conn_closed= { nullptr };
+  size_t  shared_memory_remain= { 0 };
+  char    *shared_memory_pos= { nullptr };
 #endif /* _WIN32 && !EMBEDDED_LIBRARY */
+
+private:
+  friend st_vio *internal_vio_create(uint flags);
+  friend void internal_vio_delete(st_vio *vio);
+  friend my_bool vio_reset(Vio* vio, enum_vio_type type,
+                           my_socket sd, void *ssl, uint flags);
+
+  explicit st_vio(uint flags);
+  ~st_vio();
+
+public:
+  st_vio(const st_vio&) = delete;
+  st_vio &operator=(const st_vio&) = delete;
 };
 
-#ifdef  __cplusplus
 }
-#endif
+#endif /* __cpluscplus */
 
 #endif /* vio_violite_h_ */

mysql-test/collections/default.experimental

@@ -18,8 +18,6 @@ perfschema.transaction_nested_events     # Bug#17752288 2013-12-17 agopi Fails s
 main.ssl-big @windows                    # Bug#18200216 2014-03-14 horst Fails too often in daily in PB2.
 rpl_gtid.rpl_gtid_stress_failover             # Bug#20630589  2015-03-19 agopi Originally made experimental due to bug#16409537 which was closed as not reproducible.
 
-binlog.binlog_xa_prepared_disconnect     # Bug#19502202 2015-12-02 erlend experimental while waiting for a fix
-main.xa_prepared_binlog_off              # Bug#19502202 2015-12-02 erlend experimental while waiting for a fix
 main.user_var				 # Bug#21907546 2015-12-11 erlend experimental to reduce noise
 main.locking_service			 # Bug#22304322 2015-12-11 erlend experimental to reduce noise
 main.partition_pruning			 # Bug#20597049 2015-12-11 erlend experimental to reduce noise

plugin/semisync/semisync_master_ack_receiver.cc

@@ -130,9 +130,9 @@ bool Ack_receiver::add_slave(THD *thd)
   function_enter(kWho);
 
   slave.thd= thd;
-  slave.vio= *thd->get_protocol_classic()->get_vio();
-  slave.vio.mysql_socket.m_psi= NULL;
-  slave.vio.read_timeout= 1;
+  slave.vio= thd->get_protocol_classic()->get_vio();
+  slave.vio->mysql_socket.m_psi= NULL;
+  slave.vio->read_timeout= 1;
 
   /* push_back() may throw an exception */
   try
@@ -284,7 +284,7 @@ void Ack_receiver::run()
         ulong len;
 
         net_clear(&net, 0);
-        net.vio= &m_slaves[i].vio;
+        net.vio= m_slaves[i].vio;
 
         len= my_net_read(&net);
         if (likely(len != packet_error))

plugin/semisync/semisync_master_ack_receiver.h

@@ -1,4 +1,4 @@
-/* Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2014, 2016, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -104,9 +104,9 @@ class Ack_receiver : public ReplSemiSyncBase
   struct Slave
   {
     THD *thd;
-    Vio vio;
+    Vio *vio;
 
-    my_socket sock_fd() { return vio.mysql_socket.fd; }
+    my_socket sock_fd() { return vio->mysql_socket.fd; }
     uint server_id() { return thd->server_id; }
   };