clusterer: add ability to disable replication packets dispatching

Add a new "dispatch_jobs" modparam which controls whether the processing of
replicated packets is dispatched through IPC or not.
Disabling the dispatching mechanism prevents high CPU loads caused by
the "thundering herd" problem.

Credits go to Rizwan Syed and Connex Carrier Services for supporting the
troubleshooting of this issue.

(cherry picked from commit 0c66936)

modules/clusterer/clusterer.c

@@ -56,6 +56,8 @@ extern int ping_timeout;
 extern int seed_fb_interval;
 extern int sync_timeout;
 
+int dispatch_jobs = 1;
+
 void sync_check_timer(utime_t ticks, void *param)
 {
 	cluster_info_t *cl;
@@ -1354,8 +1356,12 @@ static void bin_rcv_mod_packets(bin_packet_t *packet, int packet_type,
 			lock_stop_read(cl_list_lock);
 			packet->src_id = source_id;
 
-			if (ipc_dispatch_mod_packet(packet, cap) < 0)
-				LM_ERR("Failed to dispatch handling of module packet\n");
+			if (dispatch_jobs) {
+				if (ipc_dispatch_mod_packet(packet, cap) < 0)
+					LM_ERR("Failed to dispatch handling of module packet\n");
+			} else {
+				cap->packet_cb(packet);
+			}
 
 			return;
 		}

modules/clusterer/clusterer.h

@@ -146,6 +146,8 @@ extern enum sip_protos clusterer_proto;
 extern str cl_internal_cap;
 extern str cl_extra_cap;
 
+extern int dispatch_jobs;
+
 void sync_check_timer(utime_t ticks, void *param);
 
 void bin_rcv_cl_packets(bin_packet_t *packet, int packet_type,

modules/clusterer/clusterer_mod.c

@@ -163,6 +163,7 @@ static param_export_t params[] = {
 	{"sharing_tag",			STR_PARAM|USE_FUNC_PARAM,
 		(void*)&shtag_modparam_func},
 	{"sync_packet_size",	INT_PARAM,	&sync_packet_size	},
+	{"dispatch_jobs",		INT_PARAM,	&dispatch_jobs		},
 	{0, 0, 0}
 };
 

modules/clusterer/doc/clusterer_admin.xml

@@ -391,6 +391,37 @@ modparam("clusterer", "sync_packet_max_size", 32765)
 		</example>
         </section>
 
+        <section id="param_dispatch_jobs" xreflabel="dispatch_jobs">
+            <title><varname>dispatch_jobs</varname></title>
+            <para>
+            Enables the dispatching of jobs(processing replicated data packets)
+            from the receiving TCP worker process to free opensips workers
+            (including UDP, timer processes etc.).
+            </para>
+            <para>
+            This generally improves the performance of handling replication packets
+            in high traffic scenarios and should not be disabled.
+            </para>
+            <para>
+            Nevertheless there are cases where the "thundering herd" problem occurs
+            which causes abnormaly high CPU loads. Disabling this dispatching
+            mechanism solves such issues.
+            </para>
+            <para>
+		<emphasis>
+			Default value is <quote>1</quote> (enabled).
+		</emphasis>
+            </para>
+            <example>
+		<title>Set <varname>dispatch_jobs</varname> parameter</title>
+		<programlisting format="linespecific">
+...
+modparam("clusterer", "dispatch_jobs", 0)
+...
+		</programlisting>
+		</example>
+        </section>
+
         <section id="param_id_col" xreflabel="id_col">
             <title><varname>id_col</varname></title>
             <para>