From b93fbafed4cca67d2393f90c25710e7492b737f1 Mon Sep 17 00:00:00 2001
From: Vlad Patrascu <vladp@opensips.org>
Date: Mon, 1 Aug 2022 18:06:37 +0300
Subject: [PATCH] clusterer: add ability to disable replication packets
 dispatching

Add a new "dispatch_jobs" modparam which controls whether the processing of
replicated packets is dispatched through IPC or not.
Disabling the dispatching mechanism prevents high CPU loads caused by
the "thundering herd" problem.

Credits go to Rizwan Syed and Connex Carrier Services for supporting the
troubleshooting of this issue.

(cherry picked from commit 0c66936f3d343226930042bc463b467d8a62fde3)
---
 modules/clusterer/clusterer.c             | 10 ++++++--
 modules/clusterer/clusterer.h             |  2 ++
 modules/clusterer/clusterer_mod.c         |  1 +
 modules/clusterer/doc/clusterer_admin.xml | 31 +++++++++++++++++++++++
 4 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/modules/clusterer/clusterer.c b/modules/clusterer/clusterer.c
index d4305187fdc..d38205a44ea 100644
--- a/modules/clusterer/clusterer.c
+++ b/modules/clusterer/clusterer.c
@@ -56,6 +56,8 @@ extern int ping_timeout;
 extern int seed_fb_interval;
 extern int sync_timeout;
 
+int dispatch_jobs = 1;
+
 void sync_check_timer(utime_t ticks, void *param)
 {
 	cluster_info_t *cl;
@@ -1354,8 +1356,12 @@ static void bin_rcv_mod_packets(bin_packet_t *packet, int packet_type,
 			lock_stop_read(cl_list_lock);
 			packet->src_id = source_id;
 
-			if (ipc_dispatch_mod_packet(packet, cap) < 0)
-				LM_ERR("Failed to dispatch handling of module packet\n");
+			if (dispatch_jobs) {
+				if (ipc_dispatch_mod_packet(packet, cap) < 0)
+					LM_ERR("Failed to dispatch handling of module packet\n");
+			} else {
+				cap->packet_cb(packet);
+			}
 
 			return;
 		}
diff --git a/modules/clusterer/clusterer.h b/modules/clusterer/clusterer.h
index 45e7fedf3a7..c4f9040ccaf 100644
--- a/modules/clusterer/clusterer.h
+++ b/modules/clusterer/clusterer.h
@@ -146,6 +146,8 @@ extern enum sip_protos clusterer_proto;
 extern str cl_internal_cap;
 extern str cl_extra_cap;
 
+extern int dispatch_jobs;
+
 void sync_check_timer(utime_t ticks, void *param);
 
 void bin_rcv_cl_packets(bin_packet_t *packet, int packet_type,
diff --git a/modules/clusterer/clusterer_mod.c b/modules/clusterer/clusterer_mod.c
index a7b500d5274..f0eafdf6587 100644
--- a/modules/clusterer/clusterer_mod.c
+++ b/modules/clusterer/clusterer_mod.c
@@ -163,6 +163,7 @@ static param_export_t params[] = {
 	{"sharing_tag",			STR_PARAM|USE_FUNC_PARAM,
 		(void*)&shtag_modparam_func},
 	{"sync_packet_size",	INT_PARAM,	&sync_packet_size	},
+	{"dispatch_jobs",		INT_PARAM,	&dispatch_jobs		},
 	{0, 0, 0}
 };
 
diff --git a/modules/clusterer/doc/clusterer_admin.xml b/modules/clusterer/doc/clusterer_admin.xml
index c5e31111dde..3a1168a83db 100644
--- a/modules/clusterer/doc/clusterer_admin.xml
+++ b/modules/clusterer/doc/clusterer_admin.xml
@@ -391,6 +391,37 @@ modparam("clusterer", "sync_packet_max_size", 32765)
 		</example>
         </section>
 
+        <section id="param_dispatch_jobs" xreflabel="dispatch_jobs">
+            <title><varname>dispatch_jobs</varname></title>
+            <para>
+            Enables the dispatching of jobs(processing replicated data packets)
+            from the receiving TCP worker process to free opensips workers
+            (including UDP, timer processes etc.).
+            </para>
+            <para>
+            This generally improves the performance of handling replication packets
+            in high traffic scenarios and should not be disabled.
+            </para>
+            <para>
+            Nevertheless there are cases where the "thundering herd" problem occurs
+            which causes abnormaly high CPU loads. Disabling this dispatching
+            mechanism solves such issues.
+            </para>
+            <para>
+		<emphasis>
+			Default value is <quote>1</quote> (enabled).
+		</emphasis>
+            </para>
+            <example>
+		<title>Set <varname>dispatch_jobs</varname> parameter</title>
+		<programlisting format="linespecific">
+...
+modparam("clusterer", "dispatch_jobs", 0)
+...
+		</programlisting>
+		</example>
+        </section>
+
         <section id="param_id_col" xreflabel="id_col">
             <title><varname>id_col</varname></title>
             <para>