app/testpmd: add testpmd based sleeping

Sleep for an incremental amount of time if the fwd engine has processed
less than at least half a burst of packets (i.e 16pkts with default
setting) on a polling iteration of testpmd.

Upon detecting the threshold of >= 16 pkts on an Rxq, reset the sleep
time to zero (i.e. no sleep).

Sleep time will be increased on each iteration where the low load
conditions remain up to a total of the max sleep time which is set by
the user with the "--max-sleep-us NUM" command line argument or when in
interactive "mode set max_sleep NUM".

The default max_sleep value is 0, which means that no sleeps will occur
and the default behaviour is unchanged from previously.

Testing has been performed on AMD EPYC 7702 server with --nb-cores 12.
The results were obtained via turbostat for each individual lcore:

max_sleep 0     ====== ======== ======== ========
                  idle    4Mpps   16Mpps   Bursts
=============== ====== ======== ======== ========
C1-state %           0        0        0        0
C2-state %           0        0        0        0
% usage            100      100      100      100
Watt / core       1.14     1.18     1.19     1.14
=============== ====== ======== ======== ========

max_sleep 500   ====== ======== ======== ========
                  idle    4Mpps   16Mpps   Bursts
=============== ====== ======== ======== ========
C1-state %          99       85       74     98.6
C2-state %           0        0        0        0
% usage              1       15       26        1
Watt / core       0.04     0.18     0.28     0 04
=============== ====== ======== ======== ========

max_sleep 1000  ====== ======== ======== ========
                  idle    4Mpps   16Mpps   Bursts
=============== ====== ======== ======== ========
C1-state %           0       85       74      0.3
C2-state %          99        0        0     97.6
% usage              1       15       25        1
Watt / core       0.02     0.18     0.28     0 02
=============== ====== ======== ======== ========

On most cases, the consumption of the cores is greatly improved while
still performing zero packet loss.

Latency test has been performed on each tests above. The CPU has a C1
latency of 1us and a C2 latency of 400us. On the worst case scenario, Tx
Burst of thousands packets every seconds, the following latency in us
(micro seconds) has been observed:

=========== ==== ===== ======
max_sleep      0   500   1000
----------- ---- ----- ------
max latency   14   560   1260
min latency    5     5      6
Avg latency    7   305    617
=========== ==== ===== ======

link: https://www.github.com/torvalds/linux/tree/master/tools/power/x86/turbostat
Signed-off-by: Anthony Harivel <aharivel@redhat.com>
Signed-off-by: 0-day Robot <robot@bytheb.org>

app/test-pmd/cmdline.c

@@ -5521,6 +5521,82 @@ static cmdline_parse_inst_t cmd_set_link_check = {
 	},
 };
 
+/* *** SET MAX SLEEP *** */
+struct cmd_max_sleep_result {
+	cmdline_fixed_string_t set;
+	cmdline_fixed_string_t max_sleep;
+	uint64_t value;
+};
+
+static void
+cmd_max_sleep_set_parsed(void *parsed_result,
+		__rte_unused struct cmdline *cl,
+		__rte_unused void *data)
+{
+	struct cmd_max_sleep_result *res = parsed_result;
+	if ((!strcmp(res->set, "set"))
+		&& (!strcmp(res->max_sleep, "max_sleep")))
+		max_sleep_us = res->value;
+}
+
+static cmdline_parse_token_string_t cmd_max_sleep_set =
+	TOKEN_STRING_INITIALIZER(struct cmd_max_sleep_result,
+			set, "set");
+static cmdline_parse_token_string_t cmd_max_sleep =
+	TOKEN_STRING_INITIALIZER(struct cmd_max_sleep_result,
+			max_sleep, "max_sleep");
+static cmdline_parse_token_num_t cmd_max_sleep_value =
+	TOKEN_NUM_INITIALIZER(struct cmd_max_sleep_result,
+			value, RTE_UINT64);
+
+static cmdline_parse_inst_t cmd_set_max_sleep = {
+	.f = cmd_max_sleep_set_parsed,
+	.data = NULL,
+	.help_str = "set max_sleep <value>: Set the maximum sleep in micro seconds.",
+	.tokens = {
+		(void *)&cmd_max_sleep_set,
+		(void *)&cmd_max_sleep,
+		(void *)&cmd_max_sleep_value,
+		NULL,
+	},
+};
+
+/* *** SHOW MAX SLEEP *** */
+struct cmd_max_sleep_show_result {
+	cmdline_fixed_string_t show;
+	cmdline_fixed_string_t max_sleep;
+	uint64_t value;
+};
+
+static void
+cmd_max_sleep_show_parsed(void *parsed_result,
+		__rte_unused struct cmdline *cl,
+		__rte_unused void *data)
+{
+	struct cmd_max_sleep_show_result *res = parsed_result;
+	if ((!strcmp(res->show, "show"))
+		&& (!strcmp(res->max_sleep, "max_sleep")))
+		printf("max_sleep %"PRIu64" us\n", max_sleep_us);
+}
+
+static cmdline_parse_token_string_t cmd_show_max_sleep_show =
+	TOKEN_STRING_INITIALIZER(struct cmd_max_sleep_show_result,
+			show, "show");
+static cmdline_parse_token_string_t cmd_max_sleep_show =
+	TOKEN_STRING_INITIALIZER(struct cmd_max_sleep_show_result,
+			max_sleep, "max_sleep");
+
+static cmdline_parse_inst_t cmd_show_max_sleep = {
+	.f = cmd_max_sleep_show_parsed,
+	.data = NULL,
+	.help_str = "show max_sleep: Display the maximun sleep in micro seconds.",
+	.tokens = {
+		(void *)&cmd_show_max_sleep_show,
+		(void *)&cmd_max_sleep_show,
+		NULL,
+	},
+};
+
 /* *** SET FORWARDING MODE *** */
 struct cmd_set_fwd_mode_result {
 	cmdline_fixed_string_t set;
@@ -12769,6 +12845,8 @@ static cmdline_parse_ctx_t builtin_ctx[] = {
 	(cmdline_parse_inst_t *)&cmd_set_allmulti_mode_all,
 	(cmdline_parse_inst_t *)&cmd_set_flush_rx,
 	(cmdline_parse_inst_t *)&cmd_set_link_check,
+	(cmdline_parse_inst_t *)&cmd_set_max_sleep,
+	(cmdline_parse_inst_t *)&cmd_show_max_sleep,
 	(cmdline_parse_inst_t *)&cmd_vlan_offload,
 	(cmdline_parse_inst_t *)&cmd_vlan_tpid,
 	(cmdline_parse_inst_t *)&cmd_rx_vlan_filter_all,

app/test-pmd/parameters.c

@@ -201,6 +201,9 @@ usage(char* progname)
 	printf("  --rx-mq-mode=0xX: hexadecimal bitmask of RX mq mode can be "
 	       "enabled\n");
 	printf("  --record-core-cycles: enable measurement of CPU cycles.\n");
+	printf("  --max-sleep-us: maximum sleep time that will be requested in\n"
+	       "    microseconds per iteration of packet processing which has received zero\n"
+	       "    or a small amount of packets from the Rx queues it is polling.\n");
 	printf("  --record-burst-stats: enable display of RX and TX bursts.\n");
 	printf("  --hairpin-mode=0xXX: bitmask set the hairpin port mode.\n"
 	       "    0x10 - explicit Tx rule, 0x02 - hairpin ports paired\n"
@@ -707,6 +710,7 @@ launch_args_parse(int argc, char** argv)
 		{ "no-iova-contig",             0, 0, 0 },
 		{ "rx-mq-mode",                 1, 0, 0 },
 		{ "record-core-cycles",         0, 0, 0 },
+		{ "max-sleep-us",               1, 0, 0 },
 		{ "record-burst-stats",         0, 0, 0 },
 		{ PARAM_NUM_PROCS,              1, 0, 0 },
 		{ PARAM_PROC_ID,                1, 0, 0 },
@@ -1459,6 +1463,16 @@ launch_args_parse(int argc, char** argv)
 			}
 			if (!strcmp(lgopts[opt_idx].name, "record-core-cycles"))
 				record_core_cycles = 1;
+			if (!strcmp(lgopts[opt_idx].name, "max-sleep-us")) {
+				char *end = NULL;
+				errno = 0;
+				unsigned long sleep = strtoul(optarg, &end, 10);
+
+				if (errno != 0 || *optarg == '\0' || *end != '\0' || sleep == 0)
+					rte_exit(EXIT_FAILURE, "max-sleep-us must be > 0\n");
+
+				max_sleep_us = sleep;
+			}
 			if (!strcmp(lgopts[opt_idx].name, "record-burst-stats"))
 				record_burst_stats = 1;
 			if (!strcmp(lgopts[opt_idx].name, PARAM_NUM_PROCS))

app/test-pmd/testpmd.c

@@ -525,6 +525,11 @@ uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
  */
 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS;
 
+/*
+ * Max sleep time requested in microseconds per iteration
+ */
+uint64_t max_sleep_us;
+
 /*
  * Used to set forced link speed
  */
@@ -2255,6 +2260,8 @@ static void
 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
 {
 	struct fwd_stream **fsm;
+	uint64_t sleep_us = 0;
+	uint64_t sleep_cycles;
 	uint64_t prev_tsc;
 	streamid_t nb_fs;
 	streamid_t sm_id;
@@ -2284,6 +2291,8 @@ run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
 			pkts = (*pkt_fwd)(fs);
 			if (record_core_cycles && pkts > 0)
 				fs->busy_cycles += rte_rdtsc() - start_fs_tsc;
+			if (pkts > nb_pkt_per_burst / 2)
+				sleep_us = 0;
 		}
 #ifdef RTE_LIB_BITRATESTATS
 		if (bitrate_enabled != 0 &&
@@ -2303,10 +2312,23 @@ run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
 				latencystats_lcore_id == rte_lcore_id())
 			rte_latencystats_update();
 #endif
+		sleep_cycles = 0;
+		if (max_sleep_us) {
+			/* Check if a sleep should happen on this iteration. */
+			if (sleep_us > 0) {
+				uint64_t tsc = rte_rdtsc();
+
+				rte_delay_us_sleep(sleep_us);
+				sleep_cycles = rte_rdtsc() - tsc;
+			}
+			if (sleep_us < max_sleep_us)
+				/* Increase sleep time for next iteration. */
+				sleep_us += 1;
+		}
 		if (record_core_cycles) {
 			uint64_t tsc = rte_rdtsc();
 
-			fc->total_cycles += tsc - prev_tsc;
+			fc->total_cycles += tsc - prev_tsc - sleep_cycles;
 			prev_tsc = tsc;
 		}
 	} while (! fc->stopped);

app/test-pmd/testpmd.h

@@ -780,6 +780,8 @@ extern struct mplsoudp_decap_conf mplsoudp_decap_conf;
 
 extern enum rte_eth_rx_mq_mode rx_mq_mode;
 
+extern uint64_t max_sleep_us;
+
 extern struct rte_flow_action_conntrack conntrack_context;
 
 extern int proc_id;

doc/guides/testpmd_app_ug/run_app.rst

@@ -555,6 +555,25 @@ The command line options are:
 
     The default value is 0. Hairpin will use single port mode and implicit Tx flow mode.
 
+*   ``--max-sleep-us=N``
+
+    Set the maximum sleep in micro seconds. The default value is 0.
+
+    When `max-sleep-us` is set, the lcores running the packet forwarding may stop active polling and
+    go to sleep for an incrementing amount of time. Each time the forwarding engine processes less
+    than half a burst of packets, the sleep time will be incremented by 1 micro second, up to the
+    maximum value set by the user.
+
+    At any point, if the forwarding engine returns more than half a burst of packets, the sleep time
+    will be reset to 0.
+
+    Sleeping in the packet processing path yields back control to the kernel scheduler. The actual
+    sleep/wakeup times are not guaranteed and may differ significantly depending on system
+    configuration, allowed C-states and scheduler timer resolution (on Linux, this is controlled by
+    ``prctl(PR_SET_TIMERSLACK, nanoseconds)`` and it defaults to 10 micro seconds).
+
+    In interactive mode, the maximum sleep time can be set with ``set max_sleep N`` and displayed
+    with ``show max_sleep``.
 
 Testpmd Multi-Process Command-line Options
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

doc/guides/testpmd_app_ug/testpmd_funcs.rst

@@ -637,6 +637,12 @@ Show fec mode of a port::
 
   testpmd> show port (port_id) fec_mode
 
+show max_sleep
+~~~~~~~~~~~~~~
+
+Show max_sleep in micro seconds::
+
+  testpmd> show max_sleep
 
 Configuration Functions
 -----------------------
@@ -1802,6 +1808,13 @@ Set fec mode for a specific port::
 
   testpmd> set port (port_id) fec_mode auto|off|rs|baser
 
+Set max_sleep
+~~~~~~~~~~~~
+
+Set max_sleep in micro seconds::
+
+  testpmd> set max_sleep <us>
+
 Config Sample actions list
 ~~~~~~~~~~~~~~~~~~~~~~~~~~