From 15c6c63e45fbafe5d8b033f3c46db51e538d6826 Mon Sep 17 00:00:00 2001 From: V S Murthy Sidagam Date: Mon, 21 Sep 2020 14:05:02 +0200 Subject: [PATCH] wl#12653 A reference caching service To call a service implementation one needs to: 1. query the registry to get a reference to the service needed 2. call the service via the reference 3. call the registry to release the reference While #2 is very fast (just a function pointer call) #1 and #3 can be expensive since they'd need to interact with the registry's global structure in a read/write fashion. Hence if the above sequence is to be repeated in a quick succession it'd be beneficial to do steps #1 and #3 just once and aggregate as many #2 steps in a single sequence. This will usually mean to cache the service reference received in #1 and delay 3 for as much as possible. But since there's an active reference held to the service implementation until 3 is taken special handling is needed to make sure that: The references are released at regular intervals so changes in the registry can become effective. There is a way to mark a service implementation as "inactive" ("dying") so that until all of the active references to it are released no new ones are possible. All of the above is part of the current audit API machinery, but needs to be isolated into a separate service suite and made generally available to all services. This is what this worklog aims to implement. RB#24806 --- components/libminchassis/dynamic_loader.cc | 4 +- components/library_mysys/my_memory.cc | 3 +- components/reference_cache/CMakeLists.txt | 28 ++ components/reference_cache/cache.cc | 141 +++++++ components/reference_cache/cache.h | 63 +++ components/reference_cache/cache_allocator.h | 38 ++ components/reference_cache/channel.cc | 172 +++++++++ components/reference_cache/channel.h | 81 ++++ components/reference_cache/component.cc | 246 ++++++++++++ .../reference_cache/reference_cache_common.h | 45 +++ include/my_rcu_lock.h | 1 + .../component_malloc_allocator.h | 148 ++++++++ .../components/library_mysys/my_memory.h | 6 +- .../components/services/reference_caching.h | 359 ++++++++++++++++++ mysql-test/r/log_options_cmdline.result | 10 + .../perfschema/r/init_pfs_from_dd.result | 2 + .../suite/perfschema/t/init_pfs_from_dd.test | 1 + mysql-test/t/log_options_cmdline.test | 5 + packaging/deb-in/deb_debug.cmake | 1 + .../mysql-packagesource-test.install.in | 1 + packaging/rpm-docker/mysql.spec.in | 4 + packaging/rpm-fedora/mysql.spec.in | 5 + packaging/rpm-oel/mysql.spec.in | 5 + packaging/rpm-sles/mysql.spec.in | 5 + sql/mysqld.cc | 17 + .../components/mysql_server/CMakeLists.txt | 21 + .../mysql_server/reference_cache-t.cc | 141 +++++++ .../mysql_server/test_reference_cache.cc | 264 +++++++++++++ .../mysql_server/test_reference_cache.h | 45 +++ 29 files changed, 1856 insertions(+), 6 deletions(-) create mode 100644 components/reference_cache/CMakeLists.txt create mode 100644 components/reference_cache/cache.cc create mode 100644 components/reference_cache/cache.h create mode 100644 components/reference_cache/cache_allocator.h create mode 100644 components/reference_cache/channel.cc create mode 100644 components/reference_cache/channel.h create mode 100644 components/reference_cache/component.cc create mode 100644 components/reference_cache/reference_cache_common.h create mode 100644 include/mysql/components/library_mysys/component_malloc_allocator.h create mode 100644 include/mysql/components/services/reference_caching.h create mode 100644 unittest/gunit/components/mysql_server/reference_cache-t.cc create mode 100644 unittest/gunit/components/mysql_server/test_reference_cache.cc create mode 100644 unittest/gunit/components/mysql_server/test_reference_cache.h diff --git a/components/libminchassis/dynamic_loader.cc b/components/libminchassis/dynamic_loader.cc index 718a53983921..1183051384af 100644 --- a/components/libminchassis/dynamic_loader.cc +++ b/components/libminchassis/dynamic_loader.cc @@ -1136,7 +1136,7 @@ bool mysql_dynamic_loader_imp::unload_do_lock_provided_services( */ minimal_chassis::rwlock_scoped_lock lock = mysql_registry_imp::lock_registry_for_write(); - return mysql_dynamic_loader_imp :: + return mysql_dynamic_loader_imp:: unload_do_check_provided_services_reference_count( components_to_unload, dependency_graph, scheme_services); } @@ -1157,7 +1157,7 @@ bool mysql_dynamic_loader_imp::unload_do_lock_provided_services( @retval false success @retval true failure */ -bool mysql_dynamic_loader_imp :: +bool mysql_dynamic_loader_imp:: unload_do_check_provided_services_reference_count( const std::vector &components_to_unload, const std::map> diff --git a/components/library_mysys/my_memory.cc b/components/library_mysys/my_memory.cc index 0f753fef570c..23b6aaeb2ab7 100644 --- a/components/library_mysys/my_memory.cc +++ b/components/library_mysys/my_memory.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License, version 2.0, @@ -40,7 +40,6 @@ } while (0) #endif /* HAVE_VALGRIND */ -#define MY_ZEROFILL 32 /* fill array with zero */ #define HEADER_SIZE 32 #define MAGIC 1234 #define USER_TO_HEADER(P) ((my_memory_header *)(((char *)P) - HEADER_SIZE)) diff --git a/components/reference_cache/CMakeLists.txt b/components/reference_cache/CMakeLists.txt new file mode 100644 index 000000000000..a24bc522bcc0 --- /dev/null +++ b/components/reference_cache/CMakeLists.txt @@ -0,0 +1,28 @@ +# Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License, version 2.0, +# as published by the Free Software Foundation. +# +# This program is also distributed with certain software (including +# but not limited to OpenSSL) that is licensed under separate terms, +# as designated in a particular file or component or in included license +# documentation. The authors of MySQL hereby grant you an additional +# permission to link the program and your derivative works with the +# separately licensed software that they have included with MySQL. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License, version 2.0, for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +MYSQL_ADD_COMPONENT(reference_cache + component.cc + channel.cc + cache.cc + MODULE_ONLY + LINK_LIBRARIES library_mysys) diff --git a/components/reference_cache/cache.cc b/components/reference_cache/cache.cc new file mode 100644 index 000000000000..f36584a88296 --- /dev/null +++ b/components/reference_cache/cache.cc @@ -0,0 +1,141 @@ +/* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License, version 2.0, +as published by the Free Software Foundation. + +This program is also distributed with certain software (including +but not limited to OpenSSL) that is licensed under separate terms, +as designated in a particular file or component or in included license +documentation. The authors of MySQL hereby grant you an additional +permission to link the program and your derivative works with the +separately licensed software that they have included with MySQL. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License, version 2.0, for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ +#include "cache.h" +#include +#include +#include +#include "channel.h" + +namespace reference_caching { +cache_imp *cache_imp::create(channel_imp *channel, + SERVICE_TYPE(registry) * registry) { + assert(channel != nullptr); + return new cache_imp(channel, registry); +} + +bool cache_imp::destroy(cache_imp *cache) { + delete cache; + return false; +} + +bool cache_imp::get(unsigned service_name_index, const my_h_service **out_ref) { + bool channel_is_valid = m_channel->is_valid(); + + if (m_cache && channel_is_valid) { + // cache hit + *out_ref = m_cache[service_name_index]; + return false; + } + + // cache miss + flush(); + + m_cache = (my_h_service **)my_malloc( + KEY_mem_reference_cache, m_service_names.size() * sizeof(my_h_service *), + MY_ZEROFILL); + + my_service query("registry_query", m_registry); + + unsigned offset = 0; + for (std::string service_name : m_service_names) { + std::set cache_set; + + my_h_service_iterator iter; + if (!query->create(service_name.c_str(), &iter)) { + while (!query->is_valid(iter)) { + const char *implementation_name; + my_h_service svc; + + // can't get the name + if (query->get(iter, &implementation_name)) break; + + // not the same service + if (strncmp(implementation_name, service_name.c_str(), + service_name.length())) + break; + + // not in the ignore list + if (m_ignore_list.find(implementation_name) != m_ignore_list.end()) + continue; + + // add the reference to the list + if (!m_registry->acquire(implementation_name, &svc)) { + auto res = cache_set.insert(svc); + + /* + release the unused reference if it's a duplicate of a reference + already added + */ + if (!res.second) m_registry->release(svc); + } + + if (query->next(iter)) break; + } + query->release(iter); + } else { + // The service is not present in the registry. + continue; + } + + my_h_service *cache_row = (my_h_service *)my_malloc( + KEY_mem_reference_cache, (cache_set.size() + 1) * sizeof(my_h_service), + MY_ZEROFILL); + + my_h_service *cache_ptr = cache_row; + for (my_h_service ref : cache_set) *cache_ptr++ = ref; + + if (offset == service_name_index) *out_ref = cache_row; + + m_cache[offset++] = cache_row; + } + return false; +} + +bool cache_imp::flush() { + if (m_cache) { + unsigned offset = 0; + for (auto service_name : m_service_names) { + my_h_service *cache_row = m_cache[offset]; + if (cache_row) { + for (my_h_service *iter = cache_row; *iter; iter++) + m_registry->release(*iter); + my_free(cache_row); + m_cache[offset] = nullptr; + } + offset++; + } + my_free(m_cache); + m_cache = nullptr; + } + return false; +} + +cache_imp::cache_imp(channel_imp *channel, SERVICE_TYPE(registry) * registry) + : m_channel{channel->ref()}, m_cache{nullptr}, m_registry{registry} { + m_service_names = channel->get_service_names(); +} + +cache_imp::~cache_imp() { + flush(); + m_channel->unref(); +} +} // namespace reference_caching diff --git a/components/reference_cache/cache.h b/components/reference_cache/cache.h new file mode 100644 index 000000000000..8d8e0f9ea7a6 --- /dev/null +++ b/components/reference_cache/cache.h @@ -0,0 +1,63 @@ +/* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License, version 2.0, +as published by the Free Software Foundation. + +This program is also distributed with certain software (including +but not limited to OpenSSL) that is licensed under separate terms, +as designated in a particular file or component or in included license +documentation. The authors of MySQL hereby grant you an additional +permission to link the program and your derivative works with the +separately licensed software that they have included with MySQL. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License, version 2.0, for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#include +#include +#include +#include +#include +#include "cache_allocator.h" +#include "reference_cache_common.h" + +namespace reference_caching { + +class channel_imp; + +class cache_imp : public Cache_malloced { + public: /* top level APIs */ + static cache_imp *create(channel_imp *channel, + SERVICE_TYPE(registry) * registry); + static bool destroy(cache_imp *cache); + bool get(unsigned service_name_index, const my_h_service **ref); + bool flush(); + + public: /* utility */ + cache_imp(channel_imp *channel, SERVICE_TYPE(registry) * registry); + ~cache_imp(); + + private: + // disable copy constructors + cache_imp(const cache_imp &); + cache_imp &operator=(const cache_imp &); + + channel_imp *m_channel; + /* + This is a opaque pointer handle used to store the acquired service + implementaions handles. + */ + my_h_service **m_cache; + SERVICE_TYPE(registry) * m_registry; + service_names_set<> m_service_names; + service_names_set<> m_ignore_list; +}; + +} // namespace reference_caching diff --git a/components/reference_cache/cache_allocator.h b/components/reference_cache/cache_allocator.h new file mode 100644 index 000000000000..ff7826c539bd --- /dev/null +++ b/components/reference_cache/cache_allocator.h @@ -0,0 +1,38 @@ +/* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License, version 2.0, +as published by the Free Software Foundation. + +This program is also distributed with certain software (including +but not limited to OpenSSL) that is licensed under separate terms, +as designated in a particular file or component or in included license +documentation. The authors of MySQL hereby grant you an additional +permission to link the program and your derivative works with the +separately licensed software that they have included with MySQL. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License, version 2.0, for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#include + +#ifndef REFERENCE_CACHE_ALLOCATOR_H +#define REFERENCE_CACHE_ALLOCATOR_H + +namespace reference_caching { + +class Cache_malloced { + public: + static void *operator new(std::size_t sz); + static void operator delete(void *ptr, std::size_t sz); +}; + +} // namespace reference_caching + +#endif /* REFERENCE_CACHE_ALLOCATOR_H */ diff --git a/components/reference_cache/channel.cc b/components/reference_cache/channel.cc new file mode 100644 index 000000000000..c14633b98162 --- /dev/null +++ b/components/reference_cache/channel.cc @@ -0,0 +1,172 @@ +/* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License, version 2.0, +as published by the Free Software Foundation. + +This program is also distributed with certain software (including +but not limited to OpenSSL) that is licensed under separate terms, +as designated in a particular file or component or in included license +documentation. The authors of MySQL hereby grant you an additional +permission to link the program and your derivative works with the +separately licensed software that they have included with MySQL. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License, version 2.0, for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ +#include "channel.h" +#include +#include +#include +#include + +namespace reference_caching { + +typedef std::unordered_set, + std::equal_to, + Component_malloc_allocator> + channels_t; +static channels_t *channels; +typedef std::unordered_map< + std::string, channel_imp *, std::hash, + std::equal_to, + Component_malloc_allocator>> + channel_by_name_hash_t; +static channel_by_name_hash_t *channel_by_name_hash; +static mysql_mutex_t LOCK_channels; + +channel_imp *channel_imp::create(service_names_set<> &service_names) { + channel_imp *result = new channel_imp(service_names); + mysql_mutex_lock(&LOCK_channels); + + auto new_element = channels->insert(result); + if (!new_element.second) { + delete result; + return nullptr; + } + + for (auto service_name : service_names) { + auto new_cbyn = channel_by_name_hash->insert( + channel_by_name_hash_t::value_type(service_name, result)); + if (!new_cbyn.second) { + for (auto service_name_del : service_names) + channel_by_name_hash->erase(service_name_del); + channels->erase(new_element.first); + delete result; + return nullptr; + } + } + + mysql_mutex_unlock(&LOCK_channels); + return result->ref(); +} + +bool channel_imp::destroy(channel_imp *channel) { + bool res = true; + int ref_count; + mysql_mutex_lock(&LOCK_channels); + channel->unref(); + ref_count = channel->m_reference_count; + if (0 == ref_count) { + auto it = channels->find(channel); + if (it != channels->end()) { + channels->erase(it); + + for (auto service_name : channel->get_service_names()) + channel_by_name_hash->erase(service_name); + delete channel; + res = false; + } + } + mysql_mutex_unlock(&LOCK_channels); + return res; +} + +bool channel_imp::factory_init() { + assert(!channels); + + channels = new channels_t( + Component_malloc_allocator(KEY_mem_reference_cache)); + channel_by_name_hash = new channel_by_name_hash_t( + Component_malloc_allocator(KEY_mem_reference_cache)); + + static PSI_mutex_key key_LOCK_channels = 0; + static PSI_mutex_info all_mutex[] = { + {&key_LOCK_channels, "refcache_channel_mutex", 0, 0, + "A mutex to guard access to the channels list"}}; + + mysql_mutex_register(PSI_category, all_mutex, 1); + mysql_mutex_init(key_LOCK_channels, &LOCK_channels, NULL); + return false; +} + +bool channel_imp::factory_deinit() { + assert(channels); + mysql_mutex_lock(&LOCK_channels); + + if (channel_by_name_hash->size() || channels->size()) { + mysql_mutex_unlock(&LOCK_channels); + return true; + } + delete channel_by_name_hash; + delete channels; + channels = nullptr; + mysql_mutex_unlock(&LOCK_channels); + mysql_mutex_destroy(&LOCK_channels); + return false; +} + +channel_imp *channel_imp::channel_by_name(std::string service_name) { + channel_imp *res = nullptr; + mysql_mutex_lock(&LOCK_channels); + + auto it = channel_by_name_hash->find(service_name); + if (it != channel_by_name_hash->end()) res = it->second->ref(); + + mysql_mutex_unlock(&LOCK_channels); + return res; +} + +void channel_imp::ignore_list_copy(service_names_set<> &dest_set) { + if (m_has_ignore_list) { + mysql_mutex_lock(&LOCK_channels); + dest_set = m_ignore_list; + mysql_mutex_unlock(&LOCK_channels); + } +} + +bool channel_imp::ignore_list_add(std::string service_implementation) { + mysql_mutex_lock(&LOCK_channels); + auto ret = m_ignore_list.insert(service_implementation); + m_has_ignore_list = true; + mysql_mutex_unlock(&LOCK_channels); + return !ret.second; +} + +bool channel_imp::ignore_list_remove(std::string service_implementation) { + if (m_has_ignore_list) { + mysql_mutex_lock(&LOCK_channels); + bool ret = m_ignore_list.erase(service_implementation) != 0; + m_has_ignore_list = m_ignore_list.size() > 0; + mysql_mutex_unlock(&LOCK_channels); + return ret; + } + return true; +} + +bool channel_imp::ignore_list_clear() { + if (m_has_ignore_list) { + mysql_mutex_lock(&LOCK_channels); + m_ignore_list.clear(); + m_has_ignore_list = m_ignore_list.size(); + mysql_mutex_unlock(&LOCK_channels); + return false; + } + return true; +} +} // namespace reference_caching diff --git a/components/reference_cache/channel.h b/components/reference_cache/channel.h new file mode 100644 index 000000000000..3179cb60befd --- /dev/null +++ b/components/reference_cache/channel.h @@ -0,0 +1,81 @@ +/* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License, version 2.0, +as published by the Free Software Foundation. + +This program is also distributed with certain software (including +but not limited to OpenSSL) that is licensed under separate terms, +as designated in a particular file or component or in included license +documentation. The authors of MySQL hereby grant you an additional +permission to link the program and your derivative works with the +separately licensed software that they have included with MySQL. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License, version 2.0, for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ +#include +#include +#include +#include +#include "cache_allocator.h" +#include "reference_cache_common.h" + +namespace reference_caching { + +class channel_imp : public Cache_malloced { + public: + static channel_imp *create(service_names_set<> &service_names); + static bool destroy(channel_imp *channel); + static bool factory_init(); + static bool factory_deinit(); + static channel_imp *channel_by_name(std::string service_name); + + bool is_valid() { return m_valid; } + void set_valid(bool new_value) { + m_valid.store(new_value, std::memory_order_relaxed); + } + + service_names_set<> &get_service_names() { return m_service_names; } + + void ignore_list_copy(service_names_set<> &dest_set); + bool ignore_list_add(std::string service_implementation); + bool ignore_list_remove(std::string service_implementation); + bool ignore_list_clear(); + + bool is_alone() { return m_reference_count == 1; } + channel_imp *ref() { + m_reference_count.fetch_add(1, std::memory_order_relaxed); + return this; + } + int unref() { + return m_reference_count.fetch_sub(1, std::memory_order_relaxed); + } + channel_imp(service_names_set<> &service_names) + : m_has_ignore_list(false), m_valid{true}, m_reference_count{0} { + m_service_names = service_names; + } + ~channel_imp() {} + + bool operator==(channel_imp &other) const { + return m_service_names == other.m_service_names; + } + + private: + // disable copy constructors + channel_imp(const channel_imp &); + channel_imp &operator=(const channel_imp &); + + service_names_set<> m_service_names; + service_names_set<> m_ignore_list; + std::atomic m_has_ignore_list; + std::atomic m_valid; + std::atomic m_reference_count; +}; + +} // namespace reference_caching diff --git a/components/reference_cache/component.cc b/components/reference_cache/component.cc new file mode 100644 index 000000000000..51a81c79c95f --- /dev/null +++ b/components/reference_cache/component.cc @@ -0,0 +1,246 @@ +/* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License, version 2.0, +as published by the Free Software Foundation. + +This program is also distributed with certain software (including +but not limited to OpenSSL) that is licensed under separate terms, +as designated in a particular file or component or in included license +documentation. The authors of MySQL hereby grant you an additional +permission to link the program and your derivative works with the +separately licensed software that they have included with MySQL. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License, version 2.0, for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ +#include +#include +#include +#include +#include +#include "cache.h" +#include "channel.h" + +namespace reference_caching { + +namespace channel { + +static DEFINE_BOOL_METHOD(create, (const char *service_names[], + reference_caching_channel *out_channel)) { + try { + service_names_set<> refs; + for (unsigned idx = 0; service_names[idx]; idx++) + refs.insert(service_names[idx]); + + *out_channel = + reinterpret_cast(channel_imp::create(refs)); + return *out_channel ? false : true; + } catch (...) { + return true; + } +} + +static DEFINE_BOOL_METHOD(destroy, (reference_caching_channel channel)) { + try { + return channel_imp::destroy(reinterpret_cast(channel)); + } catch (...) { + return true; + } +} + +static DEFINE_BOOL_METHOD(invalidate, (reference_caching_channel channel)) { + try { + reinterpret_cast(channel)->set_valid(false); + return false; + } catch (...) { + return true; + } +} + +static DEFINE_BOOL_METHOD(validate, (reference_caching_channel channel)) { + try { + reinterpret_cast(channel)->set_valid(true); + return false; + } catch (...) { + return true; + } +} + +static DEFINE_BOOL_METHOD(fetch, (const char *service_name, + reference_caching_channel *out_channel)) { + try { + *out_channel = reinterpret_cast( + channel_imp::channel_by_name(service_name)); + return *out_channel ? false : true; + } catch (...) { + return true; + } +} + +} // namespace channel + +namespace cache { + +static DEFINE_BOOL_METHOD(create, (reference_caching_channel channel, + SERVICE_TYPE(registry) * registry, + reference_caching_cache *out_cache)) { + try { + cache_imp *ptr = + cache_imp::create(reinterpret_cast(channel), registry); + *out_cache = reinterpret_cast(ptr); + return false; + } catch (...) { + return true; + } +} + +static DEFINE_BOOL_METHOD(destroy, (reference_caching_cache cache)) { + try { + return cache_imp::destroy(reinterpret_cast(cache)); + } catch (...) { + return true; + } +} + +static DEFINE_BOOL_METHOD(get, (reference_caching_cache cache, + unsigned service_name_index, + const my_h_service **refs)) { + try { + return reinterpret_cast(cache)->get(service_name_index, refs); + } catch (...) { + return true; + } +} + +static DEFINE_BOOL_METHOD(flush, (reference_caching_cache cache)) { + try { + return reinterpret_cast(cache)->flush(); + } catch (...) { + return true; + } +} + +} // namespace cache + +namespace channel_ignore_list { + +static DEFINE_BOOL_METHOD(add, (reference_caching_channel channel, + const char *implementation_name)) { + try { + return reinterpret_cast(channel)->ignore_list_add( + implementation_name); + } catch (...) { + return true; + } +} + +static DEFINE_BOOL_METHOD(remove, (reference_caching_channel channel, + const char *implementation_name)) { + try { + return reinterpret_cast(channel)->ignore_list_remove( + implementation_name); + } catch (...) { + return true; + } +} + +static DEFINE_BOOL_METHOD(clear, (reference_caching_channel channel)) { + try { + return reinterpret_cast(channel)->ignore_list_clear(); + } catch (...) { + return true; + } +} + +} // namespace channel_ignore_list + +PSI_memory_key KEY_mem_reference_cache; + +void *Cache_malloced::operator new(std::size_t sz) { + return my_malloc(KEY_mem_reference_cache, sz, 0); +} + +void Cache_malloced::operator delete(void *ptr, std::size_t) { my_free(ptr); } + +static void register_instruments() { + static PSI_memory_info all_memory[] = { + {&KEY_mem_reference_cache, "reference_cache_mem", 0, 0, + "All the memory allocations for the reference cache component"}, + }; + + int count = static_cast(sizeof(all_memory) / sizeof(PSI_memory_info)); + PSI_MEMORY_CALL(register_memory)(PSI_category, all_memory, count); +} + +static mysql_service_status_t init() { + register_instruments(); + try { + return channel_imp::factory_init() ? 1 : 0; + } catch (...) { + return 1; + } +} + +static mysql_service_status_t deinit() { + try { + return channel_imp::factory_deinit() ? 1 : 0; + } catch (...) { + return 1; + } +} + +} // namespace reference_caching + +// component definition + +BEGIN_SERVICE_IMPLEMENTATION(reference_caching, reference_caching_channel) +reference_caching::channel::create, reference_caching::channel::destroy, + reference_caching::channel::invalidate, + reference_caching::channel::validate, + reference_caching::channel::fetch END_SERVICE_IMPLEMENTATION(); + +BEGIN_SERVICE_IMPLEMENTATION(reference_caching, reference_caching_cache) +reference_caching::cache::create, reference_caching::cache::destroy, + reference_caching::cache::get, + reference_caching::cache::flush END_SERVICE_IMPLEMENTATION(); + +BEGIN_SERVICE_IMPLEMENTATION(reference_caching, + reference_caching_channel_ignore_list) +reference_caching::channel_ignore_list::add, + reference_caching::channel_ignore_list::remove, + reference_caching::channel_ignore_list::clear END_SERVICE_IMPLEMENTATION(); + +BEGIN_COMPONENT_PROVIDES(reference_caching) +PROVIDES_SERVICE(reference_caching, reference_caching_channel), + PROVIDES_SERVICE(reference_caching, reference_caching_cache), + PROVIDES_SERVICE(reference_caching, reference_caching_channel_ignore_list), + END_COMPONENT_PROVIDES(); + +REQUIRES_MYSQL_MUTEX_SERVICE_PLACEHOLDER; +REQUIRES_PSI_MUTEX_SERVICE_PLACEHOLDER; +REQUIRES_PSI_MEMORY_SERVICE_PLACEHOLDER; + +/* A list of required services. */ +BEGIN_COMPONENT_REQUIRES(reference_caching) +REQUIRES_MYSQL_MUTEX_SERVICE, REQUIRES_PSI_MUTEX_SERVICE, + REQUIRES_PSI_MEMORY_SERVICE, END_COMPONENT_REQUIRES(); + +/* A list of metadata to describe the Component. */ +BEGIN_COMPONENT_METADATA(reference_caching) +METADATA("mysql.author", "Oracle Corporation"), + METADATA("mysql.license", "GPL"), END_COMPONENT_METADATA(); + +/* Declaration of the Component. */ +DECLARE_COMPONENT(reference_caching, "mysql:reference_caching") +reference_caching::init, reference_caching::deinit END_DECLARE_COMPONENT(); + +/* Defines list of Components contained in this library. Note that for now + we assume that library will have exactly one Component. */ +DECLARE_LIBRARY_COMPONENTS &COMPONENT_REF(reference_caching) + END_DECLARE_LIBRARY_COMPONENTS diff --git a/components/reference_cache/reference_cache_common.h b/components/reference_cache/reference_cache_common.h new file mode 100644 index 000000000000..5e4a88b8176b --- /dev/null +++ b/components/reference_cache/reference_cache_common.h @@ -0,0 +1,45 @@ +/* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License, version 2.0, +as published by the Free Software Foundation. + +This program is also distributed with certain software (including +but not limited to OpenSSL) that is licensed under separate terms, +as designated in a particular file or component or in included license +documentation. The authors of MySQL hereby grant you an additional +permission to link the program and your derivative works with the +separately licensed software that they have included with MySQL. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License, version 2.0, for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#ifndef REFERENCE_CACHE_COMMON_H +#define REFERENCE_CACHE_COMMON_H + +#include +#include + +namespace reference_caching { + +extern PSI_memory_key KEY_mem_reference_cache; +#define PSI_category "refcache" + +template > +class service_names_set + : public std::set> { + public: + service_names_set() + : std::set>( + Component_malloc_allocator<>(KEY_mem_reference_cache)) {} +}; + +} // namespace reference_caching + +#endif /* REFERENCE_CACHE_COMMON_H */ diff --git a/include/my_rcu_lock.h b/include/my_rcu_lock.h index 56d82d059b59..1ac87eac45f3 100644 --- a/include/my_rcu_lock.h +++ b/include/my_rcu_lock.h @@ -203,6 +203,7 @@ class MyRcuLock { */ bool write_wait_and_delete(const T *newT) { const T *oldT = this->rcu_write(newT); + if (!oldT) return false; if (!wait_for_no_readers()) { delete oldT; return false; diff --git a/include/mysql/components/library_mysys/component_malloc_allocator.h b/include/mysql/components/library_mysys/component_malloc_allocator.h new file mode 100644 index 000000000000..963987e481bd --- /dev/null +++ b/include/mysql/components/library_mysys/component_malloc_allocator.h @@ -0,0 +1,148 @@ +/* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2.0, + as published by the Free Software Foundation. + + This program is also distributed with certain software (including + but not limited to OpenSSL) that is licensed under separate terms, + as designated in a particular file or component or in included license + documentation. The authors of MySQL hereby grant you an additional + permission to link the program and your derivative works with the + separately licensed software that they have included with MySQL. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License, version 2.0, for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#ifndef COMPONENT_MALLOC_ALLOCATOR_INCLUDED +#define COMPONENT_MALLOC_ALLOCATOR_INCLUDED + +#include +#include +#include +#include +#include // std::forward +#include "my_compiler.h" + +/** + Component_malloc_allocator is a C++ STL memory allocator based on + my_malloc/my_free. + + This allows for P_S instrumentation of memory allocation done + internally by STL container classes in components. + + Example usage: + vector> + v((Component_malloc_allocator(PSI_NOT_INSTRUMENTED))); + + If the type is complicated, you can just write + Component_malloc_allocator<>(psi_key) as a shorthand for + Component_malloc_allocator(psi_key), as all + Component_malloc_allocator instances are implicitly convertible to each other + and there is a default template parameter. + + @note allocate() throws std::bad_alloc() similarly to the default + STL memory allocator. This is necessary - STL functions which allocates + memory expects it. Otherwise these functions will try to use the memory, + leading to segfaults if memory allocation was not successful. + + @note This allocator cannot be used for std::basic_string with RHEL 6/7 + because of this bug: + https://bugzilla.redhat.com/show_bug.cgi?id=1546704 + "Define _GLIBCXX_USE_CXX11_ABI gets ignored by gcc in devtoolset-7" +*/ + +template +class Component_malloc_allocator { + // This cannot be const if we want to be able to swap. + PSI_memory_key m_key; + + public: + typedef T value_type; + typedef size_t size_type; + typedef ptrdiff_t difference_type; + + typedef T *pointer; + typedef const T *const_pointer; + + typedef T &reference; + typedef const T &const_reference; + + pointer address(reference r) const { return &r; } + const_pointer address(const_reference r) const { return &r; } + + explicit Component_malloc_allocator(PSI_memory_key key) : m_key(key) {} + + template + Component_malloc_allocator(const Component_malloc_allocator &other) + : m_key(other.psi_key()) {} + + template + // on release builds assert is not evaluated. So compilation error is seen, + // hence used 'unused' attribute. + Component_malloc_allocator &operator=( + const Component_malloc_allocator &other MY_ATTRIBUTE((unused))) { + assert(m_key == other.psi_key()); // Don't swap key. + } + + pointer allocate(size_type n, const_pointer = nullptr) { + if (n == 0) return nullptr; + if (n > max_size()) throw std::bad_alloc(); + + pointer p = static_cast(my_malloc(m_key, n * sizeof(T), 0)); + if (p == nullptr) throw std::bad_alloc(); + return p; + } + + void deallocate(pointer p, size_type) { my_free(p); } + + template + void construct(U *p, Args &&... args) { + assert(p != nullptr); + try { + ::new ((void *)p) U(std::forward(args)...); + } catch (...) { + assert(false); // Constructor should not throw an exception. + } + } + + void destroy(pointer p) { + assert(p != nullptr); + try { + p->~T(); + } catch (...) { + assert(false); // Destructor should not throw an exception + } + } + + size_type max_size() const { + return std::numeric_limits::max() / sizeof(T); + } + + template + struct rebind { + typedef Component_malloc_allocator other; + }; + + PSI_memory_key psi_key() const { return m_key; } +}; + +template +bool operator==(const Component_malloc_allocator &a1, + const Component_malloc_allocator &a2) { + return a1.psi_key() == a2.psi_key(); +} + +template +bool operator!=(const Component_malloc_allocator &a1, + const Component_malloc_allocator &a2) { + return a1.psi_key() != a2.psi_key(); +} + +#endif // COMPONENT_MALLOC_ALLOCATOR_INCLUDED diff --git a/include/mysql/components/library_mysys/my_memory.h b/include/mysql/components/library_mysys/my_memory.h index 8bc0967d02f9..6fcb3f784336 100644 --- a/include/mysql/components/library_mysys/my_memory.h +++ b/include/mysql/components/library_mysys/my_memory.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License, version 2.0, @@ -34,12 +34,14 @@ linked whenever any component needs these function. */ +#define MY_ZEROFILL 32 /** fill array with zero */ + /** Allocates size bytes of memory. @param key P_S key used for memory instrumentation @param size size bytes to allocate the memory - @param flags used at the time of allocation + @param flags used at the time of allocation. Could be @ref MY_ZEROFILL */ extern "C" void *my_malloc(PSI_memory_key key, size_t size, int flags); diff --git a/include/mysql/components/services/reference_caching.h b/include/mysql/components/services/reference_caching.h new file mode 100644 index 000000000000..142afdba0d8c --- /dev/null +++ b/include/mysql/components/services/reference_caching.h @@ -0,0 +1,359 @@ +/* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License, version 2.0, +as published by the Free Software Foundation. + +This program is also distributed with certain software (including +but not limited to OpenSSL) that is licensed under separate terms, +as designated in a particular file or component or in included license +documentation. The authors of MySQL hereby grant you an additional +permission to link the program and your derivative works with the +separately licensed software that they have included with MySQL. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License, version 2.0, for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#ifndef REFERENCE_CACHING_H +#define REFERENCE_CACHING_H + +#include +#include + +/** + Handle for "producer channels". + A producer channel is the singleton used to emit events that are subject + to reference caching. + This is needed to hold a flag that will trigger forced invalidation of all + caches in a particular channel on their next access. + Multi threaded access is expected to be safe and lock less to this handle + once it's instantiated. +*/ +DEFINE_SERVICE_HANDLE(reference_caching_channel); + +/** + A handle of a "reference cache". This is created by all events producer + threads when they start and is maintained throughout their lifetimes. + Operations on that are not thread safe and are assumed to be done by a + single thread. +*/ +DEFINE_SERVICE_HANDLE(reference_caching_cache); + +/** + @ingroup group_components_services_inventory +*/ + +/** + A reference caching channel service. + + The goal of this service is to ensure that event producers spend + a MINIMAL amount of time in the emitting code when there are no consumers of + the produced events. + + Terminology + ----------- + + An event consumer is an implementation of a component service. + An event producer is a piece of code that wants to inform all currently + registered event consumers about an event. + A channel is a device that serves as a singleton of all reference caches + for all threads that are to produce events. + A reference cache is something each thread producing events must maintain + for its lifetime and use it to find the references to event consumers it + needs to call. + + Typical lifetime of the event consumers + --------------------------------------- + + At init time the event consumer will register implementations of any of + the services it's interested in receiving notifications for. + + Then optionally it might force a channel invalidation to make sure all + existing event producers will start sending notifications to it immediately. + Or it can just sit and wait for the natural producer's reference cache + refresh cycles to kick in. + + Now it is receiving notifications from all event producers as they come. + + When it wishes to no longer receive notifications it needs to mark itself + as invisible for reference cache fill-ins. In this way all reference cache + flushes will not pick this implementation up even if it's still registered + in the registry. + + Eventually all active references will be removed by the natural producers + flush cycles. + The consumer may choose to expedite this by triggering a channel + invalidation. + + As with all service implementations, when all references to the services + are released it can unload. + + Typical lifetime of the event producers + --------------------------------------- + + An event producer will usually start at init time by creating channel(s). + + Then, for each thread wishing to produce events, a reference cache must + be created and maintained until the thread will no longer be producing + events. + + Now the tread can produce events using the reference cache. + This is done by calling the get method and then iterating over the + resulting set of references and calling each one in turn as one would + normally do for registry service references. + It is assumed that the references are owned by the cache and thus they + should not be released. + + With some cyclical (e.g. at the end of each statement or something) + the event producing thread needs to flush the cache. This is to ensure + that references to event consumers are not held for very long and that + new event consumers are picked up. However flushing the cache is a relatively + expensive operation and thus a balance between the number of events produced + and the cache being flushed must be achieved. + + General remarks + --------------- + + Channels are something each event producer must have to produce events. + Channels are to be created by a single thread before the first event is ever + produced. And, once created they are to be kept until after the last event is + produced. + + Channels serve as singletons for caches and you only need one channel instance + per event producer component. + There usually will be multiple caches (one per event producing thread) per + channel. + + Creating and destroying a channel is a relatively "expensive" operation that + might involve some synchronization and should not be done frequently. + + Channels exist to allow a non-related thread to trigger invalidation of all + currently active caches on that channel. This is necessary when for example + event consumers are registered and are about to be removed. + + Invalidating a channel is a thread-safe operation that can be invoked without + synchronization at any time. + + Each channel is associated with a specific set of service names. + @note It is a set of service names, not implementation names ! + + The names are stored and used in event caches to handle all implementations + of that particular service. +*/ +BEGIN_SERVICE_DEFINITION(reference_caching_channel) +/** + Creates a channel and returns a handle for it. + + The handle created by this method must be destroyed by the invalidate + method otherwise there might be leaks. + + The channel should be created before any events are to be produced on it. + + @param service_names a list of service names that this channel will operate + on. Terminated by a NULL pointer + @param[out] out_channel placeholder for the newly created handle. + @retval false success + @retval true failure +*/ +DECLARE_BOOL_METHOD(create, (const char *service_names[], + reference_caching_channel *out_channel)); + +/** + Destroys a channel handle + + Should make sure no other thread is using the handle and no caches are + allocated on it. + + @param channel the handle to destroy + @retval false success + @retval true failure +*/ +DECLARE_BOOL_METHOD(destroy, (reference_caching_channel channel)); + +/** + Invalidate a channel + + This is thread safe to call without synchronization + and relatively fast. + + Forces an asynchronous flush on all caches that are allocated on + that channel when they're next accessed. + + @param channel the handle to destroy + @retval false success + @retval true failure +*/ +DECLARE_BOOL_METHOD(invalidate, (reference_caching_channel channel)); + +/** + Validate a channel + + This is thread safe to call without synchronization + and relatively fast. + + This function is used to validate the channel. Which helps in + getting the cached service references on that channel when they're + next accessed. + + @param channel the handle to destroy + @retval false success + @retval true failure +*/ +DECLARE_BOOL_METHOD(validate, (reference_caching_channel channel)); + +/** + Fetches a reference caching channel by name. + + Usually consumers wishing to force reference cache flush would + fetch the channel handle so they can then call invalidate on it. + + This is a relatively expensive operation as it might involve some + synchronization. + + @param service_name a service name that this channel will operate on. + @param[out] out_channel placeholder or NULL if not found. + @retval false success + @retval true failure +*/ +DECLARE_BOOL_METHOD(fetch, (const char *service_name, + reference_caching_channel *out_channel)); + +END_SERVICE_DEFINITION(reference_caching_channel) + +/** + A service to maintain an "ignore list" for reference caches. + + When a service implementation is on that ignore list it will never be + added to any reference caches when they're filled in even if the service + implementation is in the registry and implements the service + the reference caching channel is operating on. + + This is just a list of "implementations", i.e. the part of the service + implementation name after the dot. + The channel already has the name of the service so a full implementation + name can be constructed if needed. +*/ +BEGIN_SERVICE_DEFINITION(reference_caching_channel_ignore_list) +/** + Adds an implementation name to the ignore list. + + @param channel the channel to add the ignored implementation to. + @param implementation_name the second part of the service implementation + name (the part after the dot). + @retval false successfully added + @retval true error adding (e.g. the ignore is present etc. +*/ +DECLARE_BOOL_METHOD(add, (reference_caching_channel channel, + const char *implementation_name)); +/** + Remove an implementation name to the ignore list. + + @param channel the channel to remove the ignored implementation from. + @param implementation_name the second part of the service implementation + name (the part after the dot). + @retval false successfully removed + @retval true error removing or not present +*/ +DECLARE_BOOL_METHOD(remove, (reference_caching_channel channel, + const char *implementation_name)); +/** + Empty the ignore list. + + @param channel the channel to remove the ignored implementation from. + @retval false successfully removed all ignores + @retval true error removing the ignores. State unknown. +*/ +DECLARE_BOOL_METHOD(clear, (reference_caching_channel channel)); +END_SERVICE_DEFINITION(reference_caching_channel_ignore_list) + +/** + Reference cache service. + + See the reference_caching_channel service for details on how to + operate this. + + Manages thread caches for event producer threads. +*/ +BEGIN_SERVICE_DEFINITION(reference_caching_cache) +/** + Create a reference cache. + + Needs to be called before the first get() or flush() is called. + Each call to create must be paired with a call to destroy() or + there will be leaks. + + @param channel the reference cache channel to operate on. + @param registry a handle to the registry so that no time is spent taking it + @param[out] handle of the newly allocated cache. + @retval false success + @retval true failure +*/ +DECLARE_BOOL_METHOD(create, (reference_caching_channel channel, + SERVICE_TYPE(registry) * registry, + reference_caching_cache *out_cache)); +/** + Destroy a reference cache. + + Needs to be called to dispose of each cache allocated by create(). + Needs to be called after all possible calls to get() and flush(). + Once called the cache handle is invalid and should not be used anymore. + + @param channel the reference cache channel to operate on. + @param[out] handle of the newly allocated cache. + @retval false success + @retval true failure +*/ +DECLARE_BOOL_METHOD(destroy, (reference_caching_cache cache)); + +/** + Gets a set of service references for an event producer to call. + + This is the main event producer function that will be called when the + event producer needs to produce an event. + + The cache must be a valid one. And the channel too. + + If the cache is empty or invalidated (either via the channel or via a call + to flush) it will try to fill it by consulting the registry and acquiring + references to all implementations of the service the channel is created for. + Once that is done the cache will be marked as full. This a cache "miss": + a relatively expensive operation and care must be taken so it doesn't + happen very often. + + If the cache is full (not flushed) this call will return all references + stored into the cache (might be zero too). + This is a very fast operation since the cache is single-thread-use and thus + no synchronization will be done (except for checking the channel's state + of course). This is a cache "hit" and should be the normal operation of the + cache. + + @param cache the cache to use + @param service_index to get references to. Must be one of the services the + channel serves + @param[out] an array of my_h_service terminated with an empty service (0). + @retval true failure + @retval false success +*/ +DECLARE_BOOL_METHOD(get, (reference_caching_cache cache, unsigned service_index, + const my_h_service **refs)); +/** + Flush a reference cache + + This causes the reference cache supplied to be flushed. When in this state + the next call to get() will be a guaranteed cache miss and will fill in the + cache. + + @param cache the cache to flush + @retval true failure + @retval false success +*/ +DECLARE_BOOL_METHOD(flush, (reference_caching_cache cache)); +END_SERVICE_DEFINITION(reference_caching_cache) + +#endif /* REFERENCE_CACHING_H */ diff --git a/mysql-test/r/log_options_cmdline.result b/mysql-test/r/log_options_cmdline.result index f9e5b01901a6..ab84a3234b79 100644 --- a/mysql-test/r/log_options_cmdline.result +++ b/mysql-test/r/log_options_cmdline.result @@ -21,6 +21,14 @@ Note 1592 Unsafe statement written to the binary log using statement format sinc connect(localhost,unknown_user,,,MASTER_PORT,MASTER_SOCKET); ERROR 28000: Access denied for user 'unknown_user'@'localhost' (using password: NO) DROP TABLE t1; +Can't open shared library component_reference_cache +Can't open shared library component_reference_cache +Can't open shared library component_reference_cache +Can't open shared library component_reference_cache +Can't open shared library component_reference_cache +Can't open shared library component_reference_cache +Can't open shared library component_reference_cache +Can't open shared library component_reference_cache # restart: --log-error=LOG_FILE8 --lc-messages-dir=MSG_DIR --log-error-verbosity=3 connect(localhost,unknown_user,,,MASTER_PORT,MASTER_SOCKET); ERROR 28000: Access denied for user 'unknown_user'@'localhost' (using password: NO) @@ -41,6 +49,8 @@ connect(localhost,unknown_user,,,MASTER_PORT,MASTER_SOCKET); ERROR 28000: Access denied for user 'unknown_user'@'localhost' (using password: NO) SET GLOBAL log_error_services= default; UNINSTALL COMPONENT "file://component_log_sink_json"; +Can't open shared library component_reference_cache +Can't open shared library component_reference_cache Pattern "\[ERROR\].*unknown variable 'dragnet.log_error_filter_rules=a'" found INSTALL COMPONENT "file://component_log_filter_dragnet"; # restart: --no-console --log-error=LOG_FILE13 --log-error-services=log_filter_internal;log_filter_dragnet;log_sink_internal --dragnet.log-error-filter-rules= diff --git a/mysql-test/suite/perfschema/r/init_pfs_from_dd.result b/mysql-test/suite/perfschema/r/init_pfs_from_dd.result index 05f87e3e0c7d..98a478cdf7bd 100644 --- a/mysql-test/suite/perfschema/r/init_pfs_from_dd.result +++ b/mysql-test/suite/perfschema/r/init_pfs_from_dd.result @@ -19,4 +19,6 @@ ERROR 42S02: Table 'performance_schema.cond_instances' doesn't exist # Case 2: # Restart server should fail if P_S version is changed # and --innodb-read-only mode is ON. +Can't open shared library component_reference_cache +Can't open shared library component_reference_cache # restart:--debug=+d,test_p_s_metadata_version diff --git a/mysql-test/suite/perfschema/t/init_pfs_from_dd.test b/mysql-test/suite/perfschema/t/init_pfs_from_dd.test index 66a7dd0d2b0d..c67f03eb80a8 100644 --- a/mysql-test/suite/perfschema/t/init_pfs_from_dd.test +++ b/mysql-test/suite/perfschema/t/init_pfs_from_dd.test @@ -37,6 +37,7 @@ SELECT COUNT(*) FROM performance_schema.cond_instances; --let LOG_FILE = $MYSQLTEST_VARDIR/tmp/bootstrap.err --let CMD = $MYSQLD --no-defaults --innodb_dedicated_server=OFF --innodb-log-file-size=$INNODB_LOG_FILE_SIZE --debug=+d,test_p_s_metadata_version --innodb-read-only --basedir=$MYSQLD_BASEDIR --datadir=$MYSQLD_DATADIR --log-error=$LOG_FILE --secure-file-priv="" 2>&1 +--replace_regex /.*component_reference_cache.*/Can't open shared library component_reference_cache/g --error 1 --exec $CMD --enable_warnings diff --git a/mysql-test/t/log_options_cmdline.test b/mysql-test/t/log_options_cmdline.test index e8a37dd853d2..12ab7d0fd7d0 100644 --- a/mysql-test/t/log_options_cmdline.test +++ b/mysql-test/t/log_options_cmdline.test @@ -74,6 +74,7 @@ DROP TABLE t1; #----------------------------------------------------- --let LOG_FILE4 = $MYSQLTEST_VARDIR/tmp/bootstrap1.err --let CMD1= $MYSQLD --no-defaults --innodb_dedicated_server=OFF --initialize-insecure --basedir=$MYSQLD_BASEDIR --datadir=$MYSQLTEST_VARDIR/tmp/new_datadir --log-error=$LOG_FILE4 --log-error-verbosity=-1 2>&1 +--replace_regex /.*component_reference_cache.*/Can't open shared library component_reference_cache/g --exec $CMD1 --force-rmdir $MYSQLTEST_VARDIR/tmp/new_datadir @@ -81,6 +82,7 @@ DROP TABLE t1; #----------------------------------------------------- --let LOG_FILE5 = $MYSQLTEST_VARDIR/tmp/bootstrap3.err --let CMD3 = $MYSQLD --no-defaults --innodb_dedicated_server=OFF --initialize-insecure --basedir=$MYSQLD_BASEDIR --datadir=$MYSQLTEST_VARDIR/tmp/new_datadir --log-error=$LOG_FILE5 --log-error-verbosity=1 2>&1 +--replace_regex /.*component_reference_cache.*/Can't open shared library component_reference_cache/g --exec $CMD3 --force-rmdir $MYSQLTEST_VARDIR/tmp/new_datadir @@ -88,6 +90,7 @@ DROP TABLE t1; #----------------------------------------------------- --let LOG_FILE6 = $MYSQLTEST_VARDIR/tmp/bootstrap2.err --let CMD2 = $MYSQLD --no-defaults --innodb_dedicated_server=OFF --initialize-insecure --basedir=$MYSQLD_BASEDIR --datadir=$MYSQLTEST_VARDIR/tmp/new_datadir --log-error=$LOG_FILE6 --log-error-verbosity=2 2>&1 +--replace_regex /.*component_reference_cache.*/Can't open shared library component_reference_cache/g --exec $CMD2 --force-rmdir $MYSQLTEST_VARDIR/tmp/new_datadir @@ -96,6 +99,7 @@ DROP TABLE t1; #----------------------------------------------------- --let LOG_FILE7 = $MYSQLTEST_VARDIR/tmp/bootstrap4.err --let CMD4 = $MYSQLD --no-defaults --innodb_dedicated_server=OFF --initialize-insecure --basedir=$MYSQLD_BASEDIR --datadir=$MYSQLTEST_VARDIR/tmp/new_datadir --log-error=$LOG_FILE7 --log-error-verbosity=7 2>&1 +--replace_regex /.*component_reference_cache.*/Can't open shared library component_reference_cache/g --exec $CMD4 --force-rmdir $MYSQLTEST_VARDIR/tmp/new_datadir @@ -204,6 +208,7 @@ UNINSTALL COMPONENT "file://component_log_sink_json"; --let LOG_FILE12 = $MYSQLTEST_VARDIR/tmp/test12.err --let CMD12 = $MYSQLD --no-defaults --lc-messages-dir=$MYSQL_SHAREDIR --innodb_dedicated_server=OFF --initialize-insecure --basedir=$MYSQLD_BASEDIR --datadir=$MYSQLTEST_VARDIR/tmp/new_datadir --log-error=$LOG_FILE12 --dragnet.log_error_filter_rules=a 2>&1 # mysqld aborts if the option is used without loading the component +--replace_regex /.*component_reference_cache.*/Can't open shared library component_reference_cache/g --error 1 --exec $CMD12 --force-rmdir $MYSQLTEST_VARDIR/tmp/new_datadir diff --git a/packaging/deb-in/deb_debug.cmake b/packaging/deb-in/deb_debug.cmake index cdbed3c2bc86..d6740884a3bf 100644 --- a/packaging/deb-in/deb_debug.cmake +++ b/packaging/deb-in/deb_debug.cmake @@ -109,6 +109,7 @@ usr/lib/mysql/plugin/debug/component_test_host_application_signal.so usr/lib/mysql/plugin/debug/component_test_mysql_current_thread_reader.so usr/lib/mysql/plugin/debug/component_test_mysql_runtime_error.so usr/lib/mysql/plugin/debug/component_test_component_deinit.so +usr/lib/mysql/plugin/debug/component_reference_cache.so usr/lib/mysql/plugin/debug/component_udf_reg_3_func.so usr/lib/mysql/plugin/debug/component_udf_reg_avg_func.so usr/lib/mysql/plugin/debug/component_udf_reg_int_func.so diff --git a/packaging/deb-in/mysql-packagesource-test.install.in b/packaging/deb-in/mysql-packagesource-test.install.in index b77db86901f1..2c778537e1a5 100644 --- a/packaging/deb-in/mysql-packagesource-test.install.in +++ b/packaging/deb-in/mysql-packagesource-test.install.in @@ -48,6 +48,7 @@ usr/lib/mysql/plugin/component_test_host_application_signal.so usr/lib/mysql/plugin/component_test_mysql_current_thread_reader.so usr/lib/mysql/plugin/component_test_mysql_runtime_error.so usr/lib/mysql/plugin/component_test_component_deinit.so +usr/lib/mysql/plugin/component_reference_cache.so usr/lib/mysql/plugin/component_udf_reg_3_func.so usr/lib/mysql/plugin/component_udf_reg_avg_func.so usr/lib/mysql/plugin/component_udf_reg_int_func.so diff --git a/packaging/rpm-docker/mysql.spec.in b/packaging/rpm-docker/mysql.spec.in index ecad3d3070f8..d90a742be24a 100644 --- a/packaging/rpm-docker/mysql.spec.in +++ b/packaging/rpm-docker/mysql.spec.in @@ -262,6 +262,7 @@ for p in \ component_test_mysql_current_thread_reader.so \ component_test_mysql_runtime_error.so \ component_test_component_deinit.so \ + component_reference_cache.so \ component_test_page_track_component.so \ component_test_pfs_notification.so \ component_test_pfs_resource_group.so \ @@ -498,6 +499,9 @@ rm -r $(readlink var) var %dir %attr(750, mysql, mysql) /var/lib/mysql-keyring %changelog +* Mon Sep 21 2020 Murthy Sidagam - 8.0.23-1 +- Added component_reference_cache.so component + * Wed May 18 2020 Ivo Roylev - 8.0.20-1 - added keyring_oci plugin diff --git a/packaging/rpm-fedora/mysql.spec.in b/packaging/rpm-fedora/mysql.spec.in index 652ad2397579..d20ce64c9cd4 100644 --- a/packaging/rpm-fedora/mysql.spec.in +++ b/packaging/rpm-fedora/mysql.spec.in @@ -976,6 +976,7 @@ rm -r $(readlink var) var %attr(755, root, root) %{_libdir}/mysql/plugin/component_test_mysql_current_thread_reader.so %attr(755, root, root) %{_libdir}/mysql/plugin/component_test_mysql_runtime_error.so %attr(755, root, root) %{_libdir}/mysql/plugin/component_test_component_deinit.so +%attr(755, root, root) %{_libdir}/mysql/plugin/component_reference_cache.so %attr(755, root, root) %{_libdir}/mysql/plugin/component_udf_reg_3_func.so %attr(755, root, root) %{_libdir}/mysql/plugin/component_udf_reg_avg_func.so %attr(755, root, root) %{_libdir}/mysql/plugin/component_udf_reg_int_func.so @@ -1054,6 +1055,7 @@ rm -r $(readlink var) var %attr(755, root, root) %{_libdir}/mysql/plugin/debug/component_test_mysql_current_thread_reader.so %attr(755, root, root) %{_libdir}/mysql/plugin/debug/component_test_mysql_runtime_error.so %attr(755, root, root) %{_libdir}/mysql/plugin/debug/component_test_component_deinit.so +%attr(755, root, root) %{_libdir}/mysql/plugin/debug/component_reference_cache.so %attr(755, root, root) %{_libdir}/mysql/plugin/debug/component_udf_reg_3_func.so %attr(755, root, root) %{_libdir}/mysql/plugin/debug/component_udf_reg_avg_func.so %attr(755, root, root) %{_libdir}/mysql/plugin/debug/component_udf_reg_int_func.so @@ -1192,6 +1194,9 @@ rm -r $(readlink var) var %endif # with_router %changelog +* Mon Sep 21 2020 Murthy Sidagam - 8.0.23-1 +- Added component_reference_cache.so component + * Sun Jul 12 2020 Sreedhar S - 8.0.22-1 - Add a new subpackage client-plugins diff --git a/packaging/rpm-oel/mysql.spec.in b/packaging/rpm-oel/mysql.spec.in index a9b5b401f1e9..30380890c27c 100644 --- a/packaging/rpm-oel/mysql.spec.in +++ b/packaging/rpm-oel/mysql.spec.in @@ -1440,6 +1440,7 @@ fi %attr(755, root, root) %{_libdir}/mysql/plugin/component_test_mysql_current_thread_reader.so %attr(755, root, root) %{_libdir}/mysql/plugin/component_test_mysql_runtime_error.so %attr(755, root, root) %{_libdir}/mysql/plugin/component_test_component_deinit.so +%attr(755, root, root) %{_libdir}/mysql/plugin/component_reference_cache.so %attr(755, root, root) %{_libdir}/mysql/plugin/component_udf_reg_3_func.so %attr(755, root, root) %{_libdir}/mysql/plugin/component_udf_reg_avg_func.so %attr(755, root, root) %{_libdir}/mysql/plugin/component_udf_reg_int_func.so @@ -1521,6 +1522,7 @@ fi %attr(755, root, root) %{_libdir}/mysql/plugin/debug/component_test_mysql_current_thread_reader.so %attr(755, root, root) %{_libdir}/mysql/plugin/debug/component_test_mysql_runtime_error.so %attr(755, root, root) %{_libdir}/mysql/plugin/debug/component_test_component_deinit.so +%attr(755, root, root) %{_libdir}/mysql/plugin/debug/component_reference_cache.so %attr(755, root, root) %{_libdir}/mysql/plugin/debug/component_udf_reg_3_func.so %attr(755, root, root) %{_libdir}/mysql/plugin/debug/component_udf_reg_avg_func.so %attr(755, root, root) %{_libdir}/mysql/plugin/debug/component_udf_reg_int_func.so @@ -1713,6 +1715,9 @@ fi %endif # with_router %changelog +* Mon Sep 21 2020 Murthy Sidagam - 8.0.23-1 +- Added component_reference_cache.so component + * Sun Jul 12 2020 Sreedhar S - 8.0.22-1 - Add a new subpackage client-plugins diff --git a/packaging/rpm-sles/mysql.spec.in b/packaging/rpm-sles/mysql.spec.in index 24a0fb7fc884..bfaca148bb48 100644 --- a/packaging/rpm-sles/mysql.spec.in +++ b/packaging/rpm-sles/mysql.spec.in @@ -1146,6 +1146,7 @@ rm -r $(readlink var) var %attr(755, root, root) %{_libdir}/mysql/plugin/component_test_mysql_current_thread_reader.so %attr(755, root, root) %{_libdir}/mysql/plugin/component_test_mysql_runtime_error.so %attr(755, root, root) %{_libdir}/mysql/plugin/component_test_component_deinit.so +%attr(755, root, root) %{_libdir}/mysql/plugin/component_reference_cache.so %attr(755, root, root) %{_libdir}/mysql/plugin/component_udf_reg_3_func.so %attr(755, root, root) %{_libdir}/mysql/plugin/component_udf_reg_avg_func.so %attr(755, root, root) %{_libdir}/mysql/plugin/component_udf_reg_int_func.so @@ -1227,6 +1228,7 @@ rm -r $(readlink var) var %attr(755, root, root) %{_libdir}/mysql/plugin/debug/component_test_mysql_current_thread_reader.so %attr(755, root, root) %{_libdir}/mysql/plugin/debug/component_test_mysql_runtime_error.so %attr(755, root, root) %{_libdir}/mysql/plugin/debug/component_test_component_deinit.so +%attr(755, root, root) %{_libdir}/mysql/plugin/debug/component_reference_cache.so %attr(755, root, root) %{_libdir}/mysql/plugin/debug/component_udf_reg_3_func.so %attr(755, root, root) %{_libdir}/mysql/plugin/debug/component_udf_reg_avg_func.so %attr(755, root, root) %{_libdir}/mysql/plugin/debug/component_udf_reg_int_func.so @@ -1379,6 +1381,9 @@ rm -r $(readlink var) var %endif # with_router %changelog +* Mon Sep 21 2020 Murthy Sidagam - 8.0.23-1 +- Added component_reference_cache.so component + * Sun Jul 12 2020 Sreedhar S - 8.0.22-1 - Add a new subpackage client-plugins diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 5f7c898e5586..b39f2a973480 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -1879,6 +1879,12 @@ extern bool initialize_minimal_chassis(SERVICE_TYPE_NO_CONST(registry) * *registry); extern bool deinitialize_minimal_chassis(SERVICE_TYPE_NO_CONST(registry) * registry); +/* + List of components to be loaded directly using dynamic loader load. + These components should to be present in the plugin directory path. +*/ +const char *component_urns[] = {"file://component_reference_cache"}; +#define NUMBER_OF_COMPONENTS 1 /** Initializes component infrastructure by bootstrapping core component @@ -5585,6 +5591,17 @@ static int init_server_components() { if (table_def_init() | hostname_cache_init(host_cache_size)) unireg_abort(MYSQLD_ABORT_EXIT); + /* + This load function has to be called after the opt_plugin_dir variable + is initialized else it will fail to load. + The unload of these components will be done by minimal_chassis_deinit(). + So, no need to call unload of these components. + Since, it is an optional component required for GR, audit log etc. The + error check of the service availability has to be done by those + plugins/components. + */ + dynamic_loader_srv->load(component_urns, NUMBER_OF_COMPONENTS); + /* Timers not needed if only starting with --help. */ diff --git a/unittest/gunit/components/mysql_server/CMakeLists.txt b/unittest/gunit/components/mysql_server/CMakeLists.txt index 7d8d54809bcd..c1bc151aa339 100644 --- a/unittest/gunit/components/mysql_server/CMakeLists.txt +++ b/unittest/gunit/components/mysql_server/CMakeLists.txt @@ -88,6 +88,22 @@ IF(WIN32) COPY_DLL_FOR_UNITTEST("${OPENSSL_NAME}") ENDIF() +MYSQL_ADD_EXECUTABLE(reference_cache-t + reference_cache-t.cc + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugin_output_directory + ADD_TEST reference_cache-t +) +SET_TARGET_PROPERTIES(reference_cache-t PROPERTIES ENABLE_EXPORTS TRUE) +TARGET_LINK_LIBRARIES(reference_cache-t + ${GTEST_LIBRARIES} ${CMAKE_DL_LIBS} + mysql_server_component_services sql_main minchassis unit_test_common) +IF(WIN32) + TARGET_LINK_LIBRARIES(reference_cache-t my_nt_servc) +ENDIF() +ADD_DEPENDENCIES(reference_cache-t component_reference_cache + component_test_reference_cache) + + MYSQL_ADD_COMPONENT(self_required_test_component self_required_test_component.cc MODULE_ONLY @@ -103,6 +119,11 @@ MYSQL_ADD_COMPONENT(cyclic_dependency_test_component_2 MODULE_ONLY SKIP_INSTALL ) +MYSQL_ADD_COMPONENT(test_reference_cache + test_reference_cache.cc + MODULE_ONLY + SKIP_INSTALL + ) SET(EXAMPLE_COMPONENTS component_example_component1 diff --git a/unittest/gunit/components/mysql_server/reference_cache-t.cc b/unittest/gunit/components/mysql_server/reference_cache-t.cc new file mode 100644 index 000000000000..5ee28a985957 --- /dev/null +++ b/unittest/gunit/components/mysql_server/reference_cache-t.cc @@ -0,0 +1,141 @@ +/* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License, version 2.0, +as published by the Free Software Foundation. + +This program is also distributed with certain software (including +but not limited to OpenSSL) that is licensed under separate terms, +as designated in a particular file or component or in included license +documentation. The authors of MySQL hereby grant you an additional +permission to link the program and your derivative works with the +separately licensed software that they have included with MySQL. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License, version 2.0, for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#include +#include +#include +#include +#include +#include +#include "my_io.h" +#include "my_sys.h" +#include "test_reference_cache.h" +#include "unit_test_common.h" + +using registry_type_t = SERVICE_TYPE_NO_CONST(registry); +using loader_type_t = SERVICE_TYPE_NO_CONST(dynamic_loader); +using ref_cache_producer_type_t = + SERVICE_TYPE_NO_CONST(test_ref_cache_producer); +using ref_cache_consumer_type_t = + SERVICE_TYPE_NO_CONST(test_ref_cache_consumer); +SERVICE_TYPE(test_ref_cache_producer) * ref_cache_producer; +SERVICE_TYPE(test_ref_cache_consumer) * ref_cache_consumer; + +extern mysql_component_t mysql_component_mysql_server; +class reference_cache : public ::testing::Test { + protected: + void SetUp() override { + reg = NULL; + loader = NULL; + ref_cache_producer = NULL; + ref_cache_consumer = NULL; + ASSERT_FALSE(minimal_chassis_init((®), &COMPONENT_REF(mysql_server))); + ASSERT_FALSE(reg->acquire("dynamic_loader", + reinterpret_cast( + const_cast(&loader)))); + } + + void TearDown() override { + if (loader) { + ASSERT_FALSE(reg->release( + reinterpret_cast(const_cast(loader)))); + } + ASSERT_FALSE(minimal_chassis_deinit(reg, &COMPONENT_REF(mysql_server))); + } + SERVICE_TYPE_NO_CONST(registry) * reg; + SERVICE_TYPE(dynamic_loader) * loader; +}; + +TEST_F(reference_cache, try_ref_cache_load_unload) { + static const char *urns[] = {"file://component_reference_cache"}; + std::string path; + const char *urn; + make_absolute_urn(*urns, &path); + urn = path.c_str(); + ASSERT_FALSE(loader->load(&urn, 1)); + ASSERT_FALSE(loader->unload(&urn, 1)); +} + +TEST_F(reference_cache, ref_cache_components_load_unload) { + static const char *urns[] = {"file://component_reference_cache", + "file://component_test_reference_cache"}; + + std::string path; + const char *absolute_urns[2]; + for (int i = 0; i < 2; i++) + absolute_urns[i] = (char *)malloc(2046 * sizeof(char)); + make_absolute_urn(urns[0], &path); + strcpy(const_cast(absolute_urns[0]), path.c_str()); + make_absolute_urn(urns[1], &path); + strcpy(const_cast(absolute_urns[1]), path.c_str()); + + ASSERT_FALSE(loader->load(absolute_urns, 2)); + ASSERT_FALSE(reg->acquire( + "test_ref_cache_producer", + reinterpret_cast( + const_cast(&ref_cache_producer)))); + + ASSERT_FALSE(reg->acquire( + "test_ref_cache_consumer", + reinterpret_cast( + const_cast(&ref_cache_consumer)))); + + ASSERT_FALSE( + ref_cache_consumer->mysql_test_ref_cache_consumer_counter_reset()); + ASSERT_FALSE(ref_cache_consumer->mysql_test_ref_cache_consumer_counter_get()); + /* It will give one cache event */ + ASSERT_TRUE(ref_cache_producer->mysql_test_ref_cache_produce_event(0)); + ASSERT_FALSE(ref_cache_producer->mysql_test_ref_cache_flush()); + ASSERT_FALSE(ref_cache_producer->mysql_test_ref_cache_release_cache()); + ASSERT_FALSE( + ref_cache_producer->mysql_test_ref_cache_benchmark_run(0, 0, 0, 0)); + + if (ref_cache_producer) { + ASSERT_FALSE(reg->release(reinterpret_cast( + const_cast(ref_cache_producer)))); + } + + if (ref_cache_consumer) { + ASSERT_FALSE(reg->release(reinterpret_cast( + const_cast(ref_cache_consumer)))); + } + ASSERT_FALSE(loader->unload(absolute_urns, 2)); + for (int i = 0; i < 2; i++) free(const_cast(absolute_urns[i])); +} + +/* mandatory main function */ +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + + MY_INIT(argv[0]); + + char realpath_buf[FN_REFLEN]; + char basedir_buf[FN_REFLEN]; + my_realpath(realpath_buf, my_progname, 0); + size_t res_length; + dirname_part(basedir_buf, realpath_buf, &res_length); + if (res_length > 0) basedir_buf[res_length - 1] = '\0'; + my_setwd(basedir_buf, 0); + + int retval = RUN_ALL_TESTS(); + return retval; +} diff --git a/unittest/gunit/components/mysql_server/test_reference_cache.cc b/unittest/gunit/components/mysql_server/test_reference_cache.cc new file mode 100644 index 000000000000..bbddc0c712ec --- /dev/null +++ b/unittest/gunit/components/mysql_server/test_reference_cache.cc @@ -0,0 +1,264 @@ +/* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License, version 2.0, +as published by the Free Software Foundation. + +This program is also distributed with certain software (including +but not limited to OpenSSL) that is licensed under separate terms, +as designated in a particular file or component or in included license +documentation. The authors of MySQL hereby grant you an additional +permission to link the program and your derivative works with the +separately licensed software that they have included with MySQL. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License, version 2.0, for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#include "test_reference_cache.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +REQUIRES_SERVICE_PLACEHOLDER(reference_caching_channel); +REQUIRES_SERVICE_PLACEHOLDER(reference_caching_cache); +REQUIRES_SERVICE_PLACEHOLDER(reference_caching_channel_ignore_list); + +BEGIN_COMPONENT_REQUIRES(test_reference_cache) +REQUIRES_SERVICE(reference_caching_channel) +, REQUIRES_SERVICE(reference_caching_cache), + REQUIRES_SERVICE(reference_caching_channel_ignore_list), + END_COMPONENT_REQUIRES(); + +reference_caching_channel channel = nullptr; + +/** + A helper class to implement the storage into the call cache into a thread + local variable. + Make sure it's present for all threads calling the serice via the cache + + foo_cache can be: + - nonexistent (the thread local variable is null) + - invalid (the thread local variable is set, but the cache creation failed) + - valid (the thread local variable is set and the cache creation succeeded) + +*/ +class foo_cache { + reference_caching_cache cache_; + + foo_cache() : cache_(nullptr) { + assert(channel != nullptr); + if (mysql_service_reference_caching_cache->create( + channel, mysql_service_registry, &cache_)) + cache_ = nullptr; + } + + /** the thread local variable keeper */ + static foo_cache *current_cache() { + thread_local foo_cache *tl_cache_ptr = nullptr; + if (tl_cache_ptr == nullptr) tl_cache_ptr = new foo_cache(); + return tl_cache_ptr; + } + + public: + bool is_valid() { return cache_ != nullptr; } + + /** + Call the service. + Fills in the cache in the process if empty + @return: number of consumers called + */ + unsigned call(int arg) { + const my_h_service *refs; + unsigned called = 0; + if (is_valid() && + !mysql_service_reference_caching_cache->get(cache_, 0, &refs)) { + for (const my_h_service *svc = refs; *svc; svc++) { + SERVICE_TYPE(mysql_test_foo) *f = + reinterpret_cast(*svc); + if (f->emit(arg)) + break; + else + called++; + } + } + return called; // the reference wasn't valid or the service call failed + } + + /** + flush the cache. + @retval false success + @retval true failure + */ + bool flush() { + return (!is_valid() || + mysql_service_reference_caching_cache->flush(cache_)); + } + + /** helper method to get or create the thread local cache (if absent) */ + static foo_cache *get_foo_cache() { + foo_cache *ptr = current_cache(); + return ptr; + } + + /** helper method to delete the thread local cache if present */ + static void release_foo_cache() { delete current_cache(); } + + ~foo_cache() { + if (is_valid()) mysql_service_reference_caching_cache->destroy(cache_); + } +}; + +static DEFINE_BOOL_METHOD(mysql_test_ref_cache_release_cache, ()) { + foo_cache::release_foo_cache(); + return false; +} + +static DEFINE_BOOL_METHOD(mysql_test_ref_cache_produce_event, (int arg)) { + int result = 0; + foo_cache *c = foo_cache::get_foo_cache(); + if (c) { + return c->call(arg); + } + return result; +} + +static DEFINE_BOOL_METHOD(mysql_test_ref_cache_flush, ()) { + int result = 0; + foo_cache *c = foo_cache::get_foo_cache(); + if (c) { + if (c->flush()) result = 1; + } + return result; +} + +// the kill switch for the benchamrk UDFs +std::atomic kill_switch; + +/** + A benchmark UDF: spawns a number of threads and runs a number a test in them + Each benchmark does the following for each of its iterations: + 1. takes the cache (this is to measure the effect of taking the cache) + 2. if kill switch is on it exits the loop + 3. if the cache is a valid reference it calls the listeners + 4. if the cache is a valid reference flushes it on every 10th repetition + 5. it sleeps for n_sleep if the cache is not a valid reference +*/ +static DEFINE_BOOL_METHOD(mysql_test_ref_cache_benchmark_run, + (int threads, int reps, int sleep, int flush)) { + long long n_threads = 100; + long long n_reps = 100000; + long long n_sleep = 500; + long long n_flush = 10; + kill_switch = false; + + if (threads) n_threads = threads; + + if (reps) n_reps = reps; + + if (sleep) n_sleep = sleep; + + if (flush) n_flush = flush; + + std::vector thds; + for (long long i = 0; i < n_threads; i++) + thds.push_back(std::thread([n_reps, n_sleep, n_flush]() { + foo_cache *c = foo_cache::get_foo_cache(); + for (long long arg = 0; n_reps == 0 || arg < n_reps; arg++) { + /* take again to measure the effect of fetching a populated cache */ + c = foo_cache::get_foo_cache(); + + if (kill_switch.load(std::memory_order_relaxed)) break; + + if (c) { + c->call((arg % INT_MAX)); + if (n_flush && arg % n_flush == 0) c->flush(); + } else + std::this_thread::sleep_for(std::chrono::milliseconds(n_sleep)); + } + + foo_cache::release_foo_cache(); + })); + + std::for_each(thds.begin(), thds.end(), [](std::thread &t) { t.join(); }); + return 0; +} + +static DEFINE_BOOL_METHOD(mysql_test_ref_cache_benchmark_kill, ()) { + kill_switch.store(true, std::memory_order_relaxed); + return 0; +} + +static mysql_service_status_t init() { + const char *service_names[] = {"mysql_test_foo", nullptr}; + if (mysql_service_reference_caching_channel->create(service_names, &channel)) + channel = nullptr; + return 0; +} + +static mysql_service_status_t deinit() { + if (channel != nullptr) { + if (!mysql_service_reference_caching_channel->destroy(channel)) { + channel = nullptr; + } + } + return 0; +} + +std::atomic ctr; + +static DEFINE_BOOL_METHOD(mysql_test_foo_emit, (int /*arg*/)) { + ctr++; + return false; +} + +static DEFINE_BOOL_METHOD(mysql_test_ref_cache_consumer_counter_reset, ()) { + ctr = 0; + return false; +} + +static DEFINE_BOOL_METHOD(mysql_test_ref_cache_consumer_counter_get, ()) { + return ctr.load(); +} + +BEGIN_SERVICE_IMPLEMENTATION(test_reference_cache, mysql_test_foo) +mysql_test_foo_emit END_SERVICE_IMPLEMENTATION(); + +BEGIN_SERVICE_IMPLEMENTATION(test_reference_cache, test_ref_cache_producer) +mysql_test_ref_cache_produce_event, mysql_test_ref_cache_flush, + mysql_test_ref_cache_release_cache, mysql_test_ref_cache_benchmark_run, + mysql_test_ref_cache_benchmark_kill END_SERVICE_IMPLEMENTATION(); + +BEGIN_SERVICE_IMPLEMENTATION(test_reference_cache, test_ref_cache_consumer) +mysql_test_ref_cache_consumer_counter_reset, + mysql_test_ref_cache_consumer_counter_get END_SERVICE_IMPLEMENTATION(); + +BEGIN_COMPONENT_PROVIDES(test_reference_cache) +PROVIDES_SERVICE(test_reference_cache, mysql_test_foo) +, PROVIDES_SERVICE(test_reference_cache, test_ref_cache_producer), + PROVIDES_SERVICE(test_reference_cache, test_ref_cache_consumer), + END_COMPONENT_PROVIDES(); + +BEGIN_COMPONENT_METADATA(test_reference_cache) +METADATA("mysql.author", "Oracle Corporation") +, METADATA("mysql.license", "GPL"), METADATA("test_property", "1"), + END_COMPONENT_METADATA(); + +DECLARE_COMPONENT(test_reference_cache, "mysql:test_reference_cache") +init, deinit END_DECLARE_COMPONENT(); + +DECLARE_LIBRARY_COMPONENTS &COMPONENT_REF(test_reference_cache) + END_DECLARE_LIBRARY_COMPONENTS diff --git a/unittest/gunit/components/mysql_server/test_reference_cache.h b/unittest/gunit/components/mysql_server/test_reference_cache.h new file mode 100644 index 000000000000..5af946847dcd --- /dev/null +++ b/unittest/gunit/components/mysql_server/test_reference_cache.h @@ -0,0 +1,45 @@ +/* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2.0, + as published by the Free Software Foundation. + + This program is also distributed with certain software (including + but not limited to OpenSSL) that is licensed under separate terms, + as designated in a particular file or component or in included license + documentation. The authors of MySQL hereby grant you an additional + permission to link the program and your derivative works with the + separately licensed software that they have included with MySQL. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License, version 2.0, for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ +#include +#ifndef TEST_REFERENCE_CACHE_MYSQL_TEST_FOO +#define TEST_REFERENCE_CACHE_MYSQL_TEST_FOO +/** @file components/test/test_reference_cache_foo.h */ +/** a test service to test the reference caching on */ + +BEGIN_SERVICE_DEFINITION(mysql_test_foo) +DECLARE_BOOL_METHOD(emit, (int arg)); +END_SERVICE_DEFINITION(mysql_test_foo) + +BEGIN_SERVICE_DEFINITION(test_ref_cache_producer) +DECLARE_BOOL_METHOD(mysql_test_ref_cache_produce_event, (int arg)); +DECLARE_BOOL_METHOD(mysql_test_ref_cache_flush, ()); +DECLARE_BOOL_METHOD(mysql_test_ref_cache_release_cache, ()); +DECLARE_BOOL_METHOD(mysql_test_ref_cache_benchmark_run, (int, int, int, int)); +DECLARE_BOOL_METHOD(mysql_test_ref_cache_benchmark_kill, ()); +END_SERVICE_DEFINITION(test_ref_cache_producer) + +BEGIN_SERVICE_DEFINITION(test_ref_cache_consumer) +DECLARE_BOOL_METHOD(mysql_test_ref_cache_consumer_counter_reset, ()); +DECLARE_BOOL_METHOD(mysql_test_ref_cache_consumer_counter_get, ()); +END_SERVICE_DEFINITION(test_ref_cache_consumer) + +#endif /* TEST_REFERENCE_CACHE_MYSQL_TEST_FOO */