Permalink
Browse files

BUG#19502202 - SERVER SHUTDOWN HANG SEEN IN SOME INNODB

               & RPL TESTS

A MTR test binlog.binlog_xa_prepared_disconnect fails sporadically
on PB2. When the server is shutdown during the execution of the tests,
the shutdown of the mysql server doesn't happen within a specified
timeout thereby causing SIGABRT to be sent to the server and the test
aborts.
As part of the shutdown sequence, sockets are shutdown and closed in
one thread and this shall cause the POLLHUP event to be generated for
the sockets that are stuck in poll sleep state on another thread. The
generation of POLLHUP from the kernel is async and not deterministic
and time-bound.  Hence shutdown hangs and could not complete in a
specified timeout period. The patch fixes this by using ppoll in
vio_io_wait and sending pthread_kill to the thread that is stuck
on poll when the socket is undergoing shutdown.
The constructor and destructor functionality to the patch was addded
by Tor Didriksen <tor.didriksen@oracle.com>.
  • Loading branch information...
thayumanavar77 committed Jun 21, 2016
1 parent 0208744 commit 92e525ff0837c2a0da1f9805aba7eab9f48f9608
View
@@ -83,10 +83,8 @@
SERVER_STATUS_IN_TRANS_READONLY= 8192,
SERVER_SESSION_STATE_CHANGED= (1UL << 14)
};
struct st_vio;
typedef struct st_vio Vio;
typedef struct st_net {
Vio *vio;
void *vio;
unsigned char *buff,*buff_end,*write_pos,*read_pos;
my_socket fd;
unsigned long remain_in_buf,length, buf_length, where_b;
@@ -141,7 +139,7 @@
SESSION_TRACK_TRANSACTION_CHARACTERISTICS,
SESSION_TRACK_TRANSACTION_STATE
};
my_bool my_net_init(NET *net, Vio* vio);
my_bool my_net_init(NET *net, void *vio);
void my_net_local_init(NET *net);
void net_end(NET *net);
void net_clear(NET *net, my_bool check_buffer);
View
@@ -749,9 +749,13 @@ enum SERVER_STATUS_flags_enum
*/
#define ONLY_KILL_QUERY 1
struct st_vio; /* Only C */
#ifdef __cplusplus
struct st_vio;
typedef struct st_vio Vio;
#define MYSQL_VIO Vio*
#else
#define MYSQL_VIO void*
#endif
#define MAX_TINYINT_WIDTH 3 /**< Max width for a TINY w.o. sign */
#define MAX_SMALLINT_WIDTH 5 /**< Max width for a SHORT w.o. sign */
@@ -762,7 +766,7 @@ typedef struct st_vio Vio;
#define MAX_BLOB_WIDTH 16777216 /**< Default width for blob */
typedef struct st_net {
Vio *vio;
MYSQL_VIO vio;
unsigned char *buff,*buff_end,*write_pos,*read_pos;
my_socket fd; /* For Perl DBI/dbd */
/**
@@ -937,7 +941,7 @@ enum enum_session_state_type
extern "C" {
#endif
my_bool my_net_init(NET *net, Vio* vio);
my_bool my_net_init(NET *net, MYSQL_VIO vio);
void my_net_local_init(NET *net);
void net_end(NET *net);
void net_clear(NET *net, my_bool check_buffer);
View
@@ -189,7 +189,7 @@ void set_mysql_extended_error(MYSQL *mysql, int errcode, const char *sqlstate,
/* client side of the pluggable authentication */
struct st_plugin_vio_info;
void mpvio_info(Vio *vio, struct st_plugin_vio_info *info);
void mpvio_info(MYSQL_VIO vio, struct st_plugin_vio_info *info);
int run_plugin_auth(MYSQL *mysql, char *data, uint data_len,
const char *data_plugin, const char *db);
int mysql_client_plugin_init();
View
@@ -28,6 +28,15 @@
/* Simple vio interface in C; The functions are implemented in violite.c */
#if !defined(_WIN32) && !defined(__APPLE__) && !defined(__SUNPRO_CC)
#define USE_PPOLL_IN_VIO
#endif
#if defined(__cplusplus) && defined(USE_PPOLL_IN_VIO)
#include <signal.h>
#include <atomic>
#endif
#ifdef __cplusplus
extern "C" {
#endif /* __cplusplus */
@@ -38,6 +47,9 @@ void init_vio_psi_keys();
#ifdef __cplusplus
typedef struct st_vio Vio;
#define MYSQL_VIO Vio*
#else
#define MYSQL_VIO void*
#endif /* __cplusplus */
enum enum_vio_type
@@ -104,13 +116,13 @@ enum enum_vio_io_event
#define VIO_LOCALHOST 1 /* a localhost connection */
#define VIO_BUFFERED_READ 2 /* use buffered read */
#define VIO_READ_BUFFER_SIZE 16384 /* size of read buffer */
#define VIO_DESCRIPTION_SIZE 30 /* size of description */
Vio* vio_new(my_socket sd, enum enum_vio_type type, uint flags);
Vio* mysql_socket_vio_new(MYSQL_SOCKET mysql_socket, enum enum_vio_type type, uint flags);
MYSQL_VIO vio_new(my_socket sd, enum enum_vio_type type, uint flags);
MYSQL_VIO mysql_socket_vio_new(MYSQL_SOCKET mysql_socket, enum enum_vio_type type, uint flags);
#ifdef _WIN32
Vio* vio_new_win32pipe(HANDLE hPipe);
Vio* vio_new_win32shared_memory(HANDLE handle_file_map,
MYSQL_VIO vio_new_win32pipe(HANDLE hPipe);
MYSQL_VIO vio_new_win32shared_memory(HANDLE handle_file_map,
HANDLE handle_map,
HANDLE event_server_wrote,
HANDLE event_server_read,
@@ -121,41 +133,42 @@ Vio* vio_new_win32shared_memory(HANDLE handle_file_map,
#define HANDLE void *
#endif /* _WIN32 */
void vio_delete(Vio* vio);
int vio_shutdown(Vio* vio);
my_bool vio_reset(Vio* vio, enum enum_vio_type type,
void vio_delete(MYSQL_VIO vio);
int vio_shutdown(MYSQL_VIO vio);
my_bool vio_reset(MYSQL_VIO vio, enum enum_vio_type type,
my_socket sd, void *ssl, uint flags);
size_t vio_read(Vio *vio, uchar * buf, size_t size);
size_t vio_read_buff(Vio *vio, uchar * buf, size_t size);
size_t vio_write(Vio *vio, const uchar * buf, size_t size);
size_t vio_read(MYSQL_VIO vio, uchar * buf, size_t size);
size_t vio_read_buff(MYSQL_VIO vio, uchar * buf, size_t size);
size_t vio_write(MYSQL_VIO vio, const uchar * buf, size_t size);
/* setsockopt TCP_NODELAY at IPPROTO_TCP level, when possible */
int vio_fastsend(Vio *vio);
int vio_fastsend(MYSQL_VIO vio);
/* setsockopt SO_KEEPALIVE at SOL_SOCKET level, when possible */
int vio_keepalive(Vio *vio, my_bool onoff);
int vio_keepalive(MYSQL_VIO vio, my_bool onoff);
/* Whenever we should retry the last read/write operation. */
my_bool vio_should_retry(Vio *vio);
my_bool vio_should_retry(MYSQL_VIO vio);
/* Check that operation was timed out */
my_bool vio_was_timeout(Vio *vio);
my_bool vio_was_timeout(MYSQL_VIO vio);
/* Short text description of the socket for those, who are curious.. */
const char* vio_description(Vio *vio);
#define VIO_DESCRIPTION_SIZE 30 /* size of description */
void vio_description(MYSQL_VIO vio, char *buf);
/* Return the type of the connection */
enum enum_vio_type vio_type(Vio* vio);
enum enum_vio_type vio_type(MYSQL_VIO vio);
/* Return last error number */
int vio_errno(Vio*vio);
int vio_errno(MYSQL_VIO vio);
/* Get socket number */
my_socket vio_fd(Vio*vio);
my_socket vio_fd(MYSQL_VIO vio);
/* Remote peer's address and name in text form */
my_bool vio_peer_addr(Vio *vio, char *buf, uint16 *port, size_t buflen);
my_bool vio_peer_addr(MYSQL_VIO vio, char *buf, uint16 *port, size_t buflen);
/* Wait for an I/O event notification. */
int vio_io_wait(Vio *vio, enum enum_vio_io_event event, int timeout);
my_bool vio_is_connected(Vio *vio);
int vio_io_wait(MYSQL_VIO vio, enum enum_vio_io_event event, int timeout);
my_bool vio_is_connected(MYSQL_VIO vio);
#ifndef DBUG_OFF
ssize_t vio_pending(Vio *vio);
ssize_t vio_pending(MYSQL_VIO vio);
#endif
/* Set timeout for a network operation. */
int vio_timeout(Vio *vio, uint which, int timeout_sec);
int vio_timeout(MYSQL_VIO vio, uint which, int timeout_sec);
/* Connect to a peer. */
my_bool vio_socket_connect(Vio *vio, struct sockaddr *addr, socklen_t len,
my_bool vio_socket_connect(MYSQL_VIO vio, struct sockaddr *addr, socklen_t len,
int timeout);
my_bool vio_get_normalized_ip_string(const struct sockaddr *addr, size_t addr_length,
@@ -207,8 +220,8 @@ struct st_VioSSLFd
SSL_CTX *ssl_context;
};
int sslaccept(struct st_VioSSLFd*, Vio *, long timeout, unsigned long *errptr);
int sslconnect(struct st_VioSSLFd*, Vio *, long timeout, unsigned long *errptr);
int sslaccept(struct st_VioSSLFd*, MYSQL_VIO, long timeout, unsigned long *errptr);
int sslconnect(struct st_VioSSLFd*, MYSQL_VIO, long timeout, unsigned long *errptr);
struct st_VioSSLFd
*new_VioSSLConnectorFd(const char *key_file, const char *cert_file,
@@ -258,32 +271,49 @@ enum SSL_type
SSL_TYPE_SPECIFIED
};
#ifdef __cplusplus
/* HFTODO - hide this if we don't want client in embedded server */
/* This structure is for every connection on both sides */
struct st_vio
{
MYSQL_SOCKET mysql_socket; /* Instrumented socket */
my_bool localhost; /* Are we from localhost? */
bool localhost= { false }; /* Are we from localhost? */
enum_vio_type type= { NO_VIO_TYPE }; /* Type of connection */
int read_timeout= { -1 }; /* Timeout value (ms) for read ops. */
int write_timeout= { -1 }; /* Timeout value (ms) for write ops. */
int retry_count= { 1 }; /* Retry count */
bool inactive= { false }; /* Connection has been shutdown */
struct sockaddr_storage local; /* Local internet address */
struct sockaddr_storage remote; /* Remote internet address */
size_t addrLen; /* Length of remote address */
enum enum_vio_type type; /* Type of connection */
my_bool inactive; /* Connection inactive (has been shutdown) */
char desc[VIO_DESCRIPTION_SIZE]; /* Description string. This
member MUST NOT be
used directly, but only
via function
"vio_description" */
char *read_buffer; /* buffer for vio_read_buff */
char *read_pos; /* start of unfetched data in the
read buffer */
char *read_end; /* end of unfetched data */
int read_timeout; /* Timeout value (ms) for read ops. */
int write_timeout; /* Timeout value (ms) for write ops. */
int retry_count; /* Retry count */
size_t addrLen= { 0 }; /* Length of remote address */
char *read_buffer= { nullptr }; /* buffer for vio_read_buff */
char *read_pos= { nullptr }; /* start of unfetched data in the
read buffer */
char *read_end= { nullptr }; /* end of unfetched data */
#ifdef USE_PPOLL_IN_VIO
my_thread_t thread_id= { 0 }; // Thread PID
sigset_t signal_mask; // Signal mask
/*
Flag to indicate whether we are in poll or shutdown.
A true value of flag indicates either the socket
has called shutdown or it is sleeping on a poll call.
False value of this flag means that the socket is
not sleeping on a poll call.
This flag provides synchronization between two threads
one entering vio_io_wait and another entering vio_shutdown
for the same socket. If the other thread is waiting on poll
sleep, it wakes up the thread by sending a signal via
pthread_kill. Also it ensures that no other thread enters in
to a poll call if it's socket has undergone shutdown.
/*
*/
std::atomic_flag poll_shutdown_flag= ATOMIC_FLAG_INIT;
#endif
/*
VIO vtable interface to be implemented by VIO's like SSL, Socket,
Named Pipe, etc.
*/
@@ -292,49 +322,61 @@ struct st_vio
viodelete is responsible for cleaning up the VIO object by freeing
internal buffers, closing descriptors, handles.
*/
void (*viodelete)(Vio*);
int (*vioerrno)(Vio*);
size_t (*read)(Vio*, uchar *, size_t);
size_t (*write)(Vio*, const uchar *, size_t);
int (*timeout)(Vio*, uint, my_bool);
int (*viokeepalive)(Vio*, my_bool);
int (*fastsend)(Vio*);
my_bool (*peer_addr)(Vio*, char *, uint16*, size_t);
void (*in_addr)(Vio*, struct sockaddr_storage*);
my_bool (*should_retry)(Vio*);
my_bool (*was_timeout)(Vio*);
void (*viodelete)(MYSQL_VIO)= { nullptr };
int (*vioerrno)(MYSQL_VIO)= { nullptr };
size_t (*read)(MYSQL_VIO, uchar *, size_t)= { nullptr };
size_t (*write)(MYSQL_VIO, const uchar *, size_t)= { nullptr };
int (*timeout)(MYSQL_VIO, uint, my_bool)= { nullptr };
int (*viokeepalive)(MYSQL_VIO, my_bool)= { nullptr };
int (*fastsend)(MYSQL_VIO)= { nullptr };
my_bool (*peer_addr)(MYSQL_VIO, char *, uint16*, size_t)= { nullptr };
void (*in_addr)(MYSQL_VIO, struct sockaddr_storage*)= { nullptr };
my_bool (*should_retry)(MYSQL_VIO)= { nullptr };
my_bool (*was_timeout)(MYSQL_VIO)= { nullptr };
/*
vioshutdown is resposnible to shutdown/close the channel, so that no
further communications can take place, however any related buffers,
descriptors, handles can remain valid after a shutdown.
*/
int (*vioshutdown)(Vio*);
my_bool (*is_connected)(Vio*);
my_bool (*has_data) (Vio*);
int (*io_wait)(Vio*, enum enum_vio_io_event, int);
my_bool (*connect)(Vio*, struct sockaddr *, socklen_t, int);
int (*vioshutdown)(MYSQL_VIO)= { nullptr };
my_bool (*is_connected)(MYSQL_VIO)= { nullptr };
my_bool (*has_data) (MYSQL_VIO)= { nullptr };
int (*io_wait)(MYSQL_VIO, enum enum_vio_io_event, int)= { nullptr };
my_bool (*connect)(MYSQL_VIO, struct sockaddr *, socklen_t, int)= { nullptr };
#ifdef _WIN32
OVERLAPPED overlapped;
HANDLE hPipe;
OVERLAPPED overlapped = { 0 };
HANDLE hPipe { nullptr };
#endif
#ifdef HAVE_OPENSSL
void *ssl_arg;
void *ssl_arg= { nullptr };
#endif
#if defined (_WIN32) && !defined (EMBEDDED_LIBRARY)
HANDLE handle_file_map;
char *handle_map;
HANDLE event_server_wrote;
HANDLE event_server_read;
HANDLE event_client_wrote;
HANDLE event_client_read;
HANDLE event_conn_closed;
size_t shared_memory_remain;
char *shared_memory_pos;
HANDLE handle_file_map= { nullptr };
char *handle_map= { nullptr };
HANDLE event_server_wrote= { nullptr };
HANDLE event_server_read= { nullptr };
HANDLE event_client_wrote= { nullptr };
HANDLE event_client_read= { nullptr };
HANDLE event_conn_closed= { nullptr };
size_t shared_memory_remain= { 0 };
char *shared_memory_pos= { nullptr };
#endif /* _WIN32 && !EMBEDDED_LIBRARY */
private:
friend st_vio *internal_vio_create(uint flags);
friend void internal_vio_delete(st_vio *vio);
friend my_bool vio_reset(Vio* vio, enum_vio_type type,
my_socket sd, void *ssl, uint flags);
explicit st_vio(uint flags);
~st_vio();
public:
st_vio(const st_vio&) = delete;
st_vio &operator=(const st_vio&) = delete;
};
#ifdef __cplusplus
}
#endif
#endif /* __cpluscplus */
#endif /* vio_violite_h_ */
@@ -18,8 +18,6 @@ perfschema.transaction_nested_events # Bug#17752288 2013-12-17 agopi Fails s
main.ssl-big @windows # Bug#18200216 2014-03-14 horst Fails too often in daily in PB2.
rpl_gtid.rpl_gtid_stress_failover # Bug#20630589 2015-03-19 agopi Originally made experimental due to bug#16409537 which was closed as not reproducible.
binlog.binlog_xa_prepared_disconnect # Bug#19502202 2015-12-02 erlend experimental while waiting for a fix
main.xa_prepared_binlog_off # Bug#19502202 2015-12-02 erlend experimental while waiting for a fix
main.user_var # Bug#21907546 2015-12-11 erlend experimental to reduce noise
main.locking_service # Bug#22304322 2015-12-11 erlend experimental to reduce noise
main.partition_pruning # Bug#20597049 2015-12-11 erlend experimental to reduce noise
@@ -130,9 +130,9 @@ bool Ack_receiver::add_slave(THD *thd)
function_enter(kWho);
slave.thd= thd;
slave.vio= *thd->get_protocol_classic()->get_vio();
slave.vio.mysql_socket.m_psi= NULL;
slave.vio.read_timeout= 1;
slave.vio= thd->get_protocol_classic()->get_vio();
slave.vio->mysql_socket.m_psi= NULL;
slave.vio->read_timeout= 1;
/* push_back() may throw an exception */
try
@@ -284,7 +284,7 @@ void Ack_receiver::run()
ulong len;
net_clear(&net, 0);
net.vio= &m_slaves[i].vio;
net.vio= m_slaves[i].vio;
len= my_net_read(&net);
if (likely(len != packet_error))
@@ -1,4 +1,4 @@
/* Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved.
/* Copyright (c) 2014, 2016, Oracle and/or its affiliates. All rights reserved.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -104,9 +104,9 @@ class Ack_receiver : public ReplSemiSyncBase
struct Slave
{
THD *thd;
Vio vio;
Vio *vio;
my_socket sock_fd() { return vio.mysql_socket.fd; }
my_socket sock_fd() { return vio->mysql_socket.fd; }
uint server_id() { return thd->server_id; }
};
Oops, something went wrong.

0 comments on commit 92e525f

Please sign in to comment.