From a6193a380db1cf4b56558ea15e996ac6ef942895 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 15 Jul 2021 21:41:42 +0200 Subject: [PATCH] Add --shm-mlock-segment-on-creation option --- fairmq/plugins/config/Config.cxx | 43 ++++++++++++++++---------------- fairmq/shmem/Manager.h | 9 +++++++ test/transport/_options.cxx | 1 + 3 files changed, 32 insertions(+), 21 deletions(-) diff --git a/fairmq/plugins/config/Config.cxx b/fairmq/plugins/config/Config.cxx index a09a571a8..0c4a12182 100644 --- a/fairmq/plugins/config/Config.cxx +++ b/fairmq/plugins/config/Config.cxx @@ -57,27 +57,28 @@ Plugin::ProgOptions ConfigPluginProgramOptions() namespace po = boost::program_options; auto pluginOptions = po::options_description{"FairMQ device options"}; pluginOptions.add_options() - ("id", po::value()->default_value(""), "Device ID.") - ("io-threads", po::value()->default_value(1), "Number of I/O threads.") - ("transport", po::value()->default_value("zeromq"), "Transport ('zeromq'/'shmem').") - ("network-interface", po::value()->default_value("default"), "Network interface to bind on (e.g. eth0, ib0..., default will try to detect the interface of the default route).") - ("init-timeout", po::value()->default_value(120), "Timeout for the initialization in seconds (when expecting dynamic initialization).") - ("max-run-time", po::value()->default_value(0), "Maximum runtime for the Running state handler, after which state will change to Ready (in seconds, 0 for no limit).") - ("print-channels", po::value()->implicit_value(true), "Print registered channel endpoints in a machine-readable format (::)") - ("shm-segment-size", po::value()->default_value(2ULL << 30), "Shared memory: size of the shared memory segment (in bytes).") - ("shm-allocation", po::value()->default_value("rbtree_best_fit"), "Shared memory allocation algorithm: rbtree_best_fit/simple_seq_fit.") - ("shm-segment-id", po::value()->default_value(0), "EXPERIMENTAL: Shared memory segment id for message creation.") - ("shm-mlock-segment", po::value()->default_value(false), "Shared memory: mlock the shared memory segment after initialization.") - ("shm-zero-segment", po::value()->default_value(false), "Shared memory: zero the shared memory segment memory after initialization.") - ("shm-throw-bad-alloc", po::value()->default_value(true), "Throw a fair::mq::MessageBadAlloc if cannot allocate a message (retry if false).") - ("shm-monitor", po::value()->default_value(true), "Shared memory: run monitor daemon.") - ("shm-no-cleanup", po::value()->default_value(false), "Shared memory: do not cleanup the memory when last device leaves.") - ("ofi-size-hint", po::value()->default_value(0), "EXPERIMENTAL: OFI size hint for the allocator.") - ("rate", po::value()->default_value(0.), "Rate for conditional run loop (Hz).") - ("session", po::value()->default_value("default"), "Session name.") - ("config-key", po::value(), "Use provided value instead of device id for fetching the configuration from JSON file.") - ("mq-config", po::value(), "JSON input as file.") - ("channel-config", po::value>()->multitoken()->composing(), "Configuration of single or multiple channel(s) by comma separated key=value list"); + ("id", po::value()->default_value(""), "Device ID.") + ("io-threads", po::value()->default_value(1), "Number of I/O threads.") + ("transport", po::value()->default_value("zeromq"), "Transport ('zeromq'/'shmem').") + ("network-interface", po::value()->default_value("default"), "Network interface to bind on (e.g. eth0, ib0..., default will try to detect the interface of the default route).") + ("init-timeout", po::value()->default_value(120), "Timeout for the initialization in seconds (when expecting dynamic initialization).") + ("max-run-time", po::value()->default_value(0), "Maximum runtime for the Running state handler, after which state will change to Ready (in seconds, 0 for no limit).") + ("print-channels", po::value()->implicit_value(true), "Print registered channel endpoints in a machine-readable format (::)") + ("shm-segment-size", po::value()->default_value(2ULL << 30), "Shared memory: size of the shared memory segment (in bytes).") + ("shm-allocation", po::value()->default_value("rbtree_best_fit"), "Shared memory allocation algorithm: rbtree_best_fit/simple_seq_fit.") + ("shm-segment-id", po::value()->default_value(0), "EXPERIMENTAL: Shared memory segment id for message creation.") + ("shm-mlock-segment", po::value()->default_value(false), "Shared memory: mlock the shared memory segment after initialization.") + ("shm-mlock-segment-on-creation", po::value()->default_value(false), "Shared memory: mlock the shared memory segment only once when created.") + ("shm-zero-segment", po::value()->default_value(false), "Shared memory: zero the shared memory segment memory after initialization.") + ("shm-throw-bad-alloc", po::value()->default_value(true), "Throw a fair::mq::MessageBadAlloc if cannot allocate a message (retry if false).") + ("shm-monitor", po::value()->default_value(true), "Shared memory: run monitor daemon.") + ("shm-no-cleanup", po::value()->default_value(false), "Shared memory: do not cleanup the memory when last device leaves.") + ("ofi-size-hint", po::value()->default_value(0), "EXPERIMENTAL: OFI size hint for the allocator.") + ("rate", po::value()->default_value(0.), "Rate for conditional run loop (Hz).") + ("session", po::value()->default_value("default"), "Session name.") + ("config-key", po::value(), "Use provided value instead of device id for fetching the configuration from JSON file.") + ("mq-config", po::value(), "JSON input as file.") + ("channel-config", po::value>()->multitoken()->composing(), "Configuration of single or multiple channel(s) by comma separated key=value list"); return pluginOptions; } diff --git a/fairmq/shmem/Manager.h b/fairmq/shmem/Manager.h index ac03eb8ff..9be47755f 100644 --- a/fairmq/shmem/Manager.h +++ b/fairmq/shmem/Manager.h @@ -90,11 +90,13 @@ class Manager LOG(debug) << "Generated shmid '" << fShmId << "' out of session id '" << sessionName << "'."; bool mlockSegment = false; + bool mlockSegmentOnCreation = false; bool zeroSegment = false; bool autolaunchMonitor = false; std::string allocationAlgorithm("rbtree_best_fit"); if (config) { mlockSegment = config->GetProperty("shm-mlock-segment", mlockSegment); + mlockSegmentOnCreation = config->GetProperty("shm-mlock-segment-on-creation", mlockSegmentOnCreation); zeroSegment = config->GetProperty("shm-zero-segment", zeroSegment); autolaunchMonitor = config->GetProperty("shm-monitor", autolaunchMonitor); allocationAlgorithm = config->GetProperty("shm-allocation", allocationAlgorithm); @@ -162,6 +164,13 @@ class Manager } ss << "Created "; (fEventCounter->fCount)++; + if (mlockSegmentOnCreation) { + LOG(error) << "Locking the managed segment memory pages..."; + if (mlock(boost::apply_visitor(SegmentAddress(), fSegments.at(fSegmentId)), boost::apply_visitor(SegmentSize(), fSegments.at(fSegmentId))) == -1) { + LOG(error) << "Could not lock the managed segment memory. Code: " << errno << ", reason: " << strerror(errno); + } + LOG(error) << "Successfully locked the managed segment memory pages."; + } } else { op = "open"; // found segment with the given id, opening diff --git a/test/transport/_options.cxx b/test/transport/_options.cxx index 9d0ed855d..c8df2d686 100644 --- a/test/transport/_options.cxx +++ b/test/transport/_options.cxx @@ -75,6 +75,7 @@ void ZeroingAndMlock(const string& transport) config.SetProperty("shm-segment-size", 16384); config.SetProperty("shm-zero-segment", true); config.SetProperty("shm-mlock-segment", true); + config.SetProperty("shm-mlock-segment-on-creation", true); auto factory = FairMQTransportFactory::CreateTransportFactory(transport, fair::mq::tools::Uuid(), &config);