filed-conf: make num_workers configurable

bareos · Nov 6, 2023 · fa65c38 · fa65c38
1 parent 60548ab
commit fa65c38
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 2 deletions.
diff --git a/core/src/filed/backup.cc b/core/src/filed/backup.cc
@@ -1082,6 +1082,7 @@ static inline bool SendPlainData(b_ctx& bctx)
   auto file_size = bctx.ff_pkt->statp.st_size;
   auto* flags = bctx.ff_pkt->flags;
 
+  const std::size_t num_workers = me->MaxWorkersPerJob;
   // Currently we do not support encryption while doing
   // parallel sending/checksumming/compression/etc.
   // This is mostly because EncryptData() is weird!
@@ -1090,6 +1091,10 @@ static inline bool SendPlainData(b_ctx& bctx)
   // Setting up the parallel pipeline is not worth it for small files.
   if (file_size < 128 * 1024) { return SendPlainDataSerially(bctx); }
 
+  // setting maximum worker threads to 0 means that you do not want
+  // multithreading, so just use the serial code path for now.
+  if (num_workers == 0) { return SendPlainDataSerially(bctx); }
+
   bool retval = false;
   BareosSocket* sd = bctx.jcr->store_bsock;
 
@@ -1114,8 +1119,8 @@ static inline bool SendPlainData(b_ctx& bctx)
   }
 
   auto& threadpool = bctx.jcr->fd_impl->threads;
-  work_group compute_group(20);
-  const std::size_t num_workers = std::thread::hardware_concurrency() / 2;
+
+  work_group compute_group(num_workers * 3);
 
   // FIXME(ssura): this should become a std::latch once C++20 arrives
   std::condition_variable compute_fin;

diff --git a/core/src/filed/filed_conf.cc b/core/src/filed/filed_conf.cc
@@ -92,6 +92,7 @@ static ResourceItem cli_items[] = {
   {"PluginNames", CFG_TYPE_PLUGIN_NAMES, ITEM(res_client, plugin_names), 0, 0, NULL, NULL, NULL},
   {"ScriptsDirectory", CFG_TYPE_DIR, ITEM(res_client, scripts_directory), 0, 0, NULL, NULL, NULL},
   {"MaximumConcurrentJobs", CFG_TYPE_PINT32, ITEM(res_client, MaxConcurrentJobs), 0, CFG_ITEM_DEFAULT, "20", NULL, NULL},
+  {"MaximumWorkersPerJob", CFG_TYPE_PINT32, ITEM(res_client, MaxWorkersPerJob), 0, CFG_ITEM_DEFAULT, "2", NULL, NULL},
   {"Messages", CFG_TYPE_RES, ITEM(res_client, messages), R_MSGS, 0, NULL, NULL, NULL},
   {"SdConnectTimeout", CFG_TYPE_TIME, ITEM(res_client, SDConnectTimeout), 0, CFG_ITEM_DEFAULT, "1800" /* 30 minutes */, NULL, NULL},
   {"HeartbeatInterval", CFG_TYPE_TIME, ITEM(res_client, heartbeat_interval), 0, CFG_ITEM_DEFAULT, "0", NULL, NULL},

diff --git a/core/src/filed/filed_conf.h b/core/src/filed/filed_conf.h
@@ -97,6 +97,7 @@ class ClientResource
   char* scripts_directory = nullptr;
   MessagesResource* messages = nullptr; /* Daemon message handler */
   uint32_t MaxConcurrentJobs = 0;
+  uint32_t MaxWorkersPerJob{0};
   utime_t SDConnectTimeout = {0};       /* Timeout in seconds */
   utime_t heartbeat_interval = {0};     /* Interval to send heartbeats */
   uint32_t max_network_buffer_size = 0; /* Max network buf size */

diff --git a/docs/manuals/source/include/autogenerated/bareos-fd-config-schema.json b/docs/manuals/source/include/autogenerated/bareos-fd-config-schema.json
@@ -237,6 +237,12 @@
           "default_value": "20",
           "equals": true
         },
+        "MaximumWorkersPerJob": {
+          "datatype": "PINT32",
+          "code": 0,
+          "default_value": "2",
+          "equals": true
+        },
         "Messages": {
           "datatype": "RES",
           "code": 2,
@@ -521,6 +527,12 @@
           "default_value": "20",
           "equals": true
         },
+        "MaximumWorkersPerJob": {
+          "datatype": "PINT32",
+          "code": 0,
+          "default_value": "2",
+          "equals": true
+        },
         "Messages": {
           "datatype": "RES",
           "code": 2,