diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 41cc525958a7f..f52dcf1ec6a6d 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -999,6 +999,11 @@ OPTION(bluestore_bluefs_reclaim_ratio, OPT_FLOAT, .20) // how much to reclaim at // get the serial number of Intel(R) Fultondale NVMe controllers. // Example: // bluestore_block_path = spdk:55cd2e404bd73932 +// If you want to run multiple SPDK instances per node, you must specify the +// amount of memory per socket each instance will use. +OPTION(bluestore_spdk_socket_mem, OPT_STR, "512,512") +// A hexadecimal bit mask of the cores to run on. Note the core numbering can change between platforms and should be determined beforehand. +OPTION(bluestore_spdk_coremask, OPT_STR, "0x3") OPTION(bluestore_block_path, OPT_STR, "") OPTION(bluestore_block_size, OPT_U64, 10 * 1024*1024*1024) // 10gb for testing OPTION(bluestore_block_create, OPT_BOOL, true) diff --git a/src/os/bluestore/NVMEDevice.cc b/src/os/bluestore/NVMEDevice.cc index 13b01f0a20d6c..8f983bfbec853 100644 --- a/src/os/bluestore/NVMEDevice.cc +++ b/src/os/bluestore/NVMEDevice.cc @@ -626,6 +626,17 @@ int NVMEManager::try_get(const string &sn_tag, SharedDriverData **driver) { Mutex::Locker l(lock); int r = 0; + + string prefix = "--file-prefix="; + string sock_mem = "--socket-mem="; + string coremask = "-c "; + prefix += sn_tag; + sock_mem += g_conf->bluestore_spdk_socket_mem; + coremask += g_conf->bluestore_spdk_coremask; + char *prefix_arg = (char *)prefix.c_str(); + char *sock_mem_arg = (char *)sock_mem.c_str(); + char *coremask_arg = (char *)coremask.c_str(); + if (sn_tag.empty()) { r = -ENOENT; derr << __func__ << " empty serial number: " << cpp_strerror(r) << dendl; @@ -642,11 +653,13 @@ int NVMEManager::try_get(const string &sn_tag, SharedDriverData **driver) if (!init) { init = true; dpdk_thread = std::thread( - [this]() { + [this, prefix_arg, sock_mem_arg, coremask_arg]() { static const char *ealargs[] = { "ceph-osd", - "-c 0x3", /* This must be the second parameter. It is overwritten by index in main(). */ + coremask_arg, /* This must be the second parameter. It is overwritten by index in main(). */ "-n 4", + socket_mem_arg, + prefix_arg }; int r = rte_eal_init(sizeof(ealargs) / sizeof(ealargs[0]), (char **)(void *)(uintptr_t)ealargs);