Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

dpdk: add warning in configuration phase and minor DPDK refactoring v2 #10449

Closed
wants to merge 4 commits into from
Closed

dpdk: add warning in configuration phase and minor DPDK refactoring v2 #10449

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6e7d4d6
dpdk: add missing SCEnter macros to functions with SCReturns
Feb 11, 2024
5f633e5
dpdk: split out the configure function into smaller ones
Feb 11, 2024
2440290
dpdk: refactor log messages to have the same error format
Feb 11, 2024
6a37359
dpdk: add a user warning on the change of RX/TX descriptors
Feb 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
155 changes: 90 additions & 65 deletions src/runmode-dpdk.c
View file
Open in desktop
Expand Up @@ -349,7 +349,7 @@ static void ConfigSetIface(DPDKIfaceConfig *iconf, const char *entry_str)

retval = rte_eth_dev_get_port_by_name(entry_str, &iconf->port_id);
if (retval < 0)
FatalError("Interface \"%s\": %s", entry_str, rte_strerror(-retval));
FatalError("Name of the interface (%s) is invalid: %s", entry_str, rte_strerror(-retval));
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this message suggests the name itself is invalid, but is that what is meant here? A name can be valid w/o being present in the system?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, it was meant to say that the interface does not exist. Will reword.


strlcpy(iconf->iface, entry_str, sizeof(iconf->iface));
SCReturn;
Expand Down Expand Up @@ -626,8 +626,8 @@ static int ConfigSetCopyIface(DPDKIfaceConfig *iconf, const char *entry_str)

retval = rte_eth_dev_get_port_by_name(entry_str, &iconf->out_port_id);
if (retval < 0) {
SCLogError("%s: name of the copy interface (%s) is invalid (err %s)", iconf->iface,
entry_str, rte_strerror(-retval));
SCLogError("%s: name of the copy interface (%s) is invalid: %s", iconf->iface, entry_str,
rte_strerror(-retval));
SCReturnInt(retval);
}

Expand Down Expand Up @@ -1070,6 +1070,7 @@ static void DumpRXOffloadCapabilities(const uint64_t rx_offld_capa)

static int DeviceValidateMTU(const DPDKIfaceConfig *iconf, const struct rte_eth_dev_info *dev_info)
{
SCEnter();
if (iconf->mtu > dev_info->max_mtu || iconf->mtu < dev_info->min_mtu) {
SCLogError("%s: MTU out of bounds. "
"Min MTU: %" PRIu16 " Max MTU: %" PRIu16,
Expand Down Expand Up @@ -1122,27 +1123,17 @@ static int32_t DeviceSetSocketID(uint16_t port_id, int32_t *socket_id)
return retval;
}

static void DeviceInitPortConf(const DPDKIfaceConfig *iconf,
const struct rte_eth_dev_info *dev_info, struct rte_eth_conf *port_conf)
static void PortConfSetInterruptMode(const DPDKIfaceConfig *iconf, struct rte_eth_conf *port_conf)
{
DumpRXOffloadCapabilities(dev_info->rx_offload_capa);
*port_conf = (struct rte_eth_conf){
.rxmode = {
.mq_mode = RTE_ETH_MQ_RX_NONE,
.offloads = 0, // turn every offload off to prevent any packet modification
},
.txmode = {
.mq_mode = RTE_ETH_MQ_TX_NONE,
.offloads = 0,
},
};

SCLogConfig("%s: interrupt mode is %s", iconf->iface,
iconf->flags & DPDK_IRQ_MODE ? "enabled" : "disabled");
if (iconf->flags & DPDK_IRQ_MODE)
port_conf->intr_conf.rxq = 1;
}

// configure RX offloads
static void PortConfSetRSSConf(const DPDKIfaceConfig *iconf,
const struct rte_eth_dev_info *dev_info, struct rte_eth_conf *port_conf)
{
if (dev_info->rx_offload_capa & RTE_ETH_RX_OFFLOAD_RSS_HASH) {
if (iconf->nb_rx_queues > 1) {
SCLogConfig("%s: RSS enabled for %d queues", iconf->iface, iconf->nb_rx_queues);
Expand Down Expand Up @@ -1178,7 +1169,11 @@ static void DeviceInitPortConf(const DPDKIfaceConfig *iconf,
} else {
SCLogConfig("%s: RSS not supported", iconf->iface);
}
}

static void PortConfSetChsumOffload(const DPDKIfaceConfig *iconf,
const struct rte_eth_dev_info *dev_info, struct rte_eth_conf *port_conf)
{
if (iconf->checksum_mode == CHECKSUM_VALIDATION_DISABLE) {
SCLogConfig("%s: checksum validation disabled", iconf->iface);
} else if ((dev_info->rx_offload_capa & RTE_ETH_RX_OFFLOAD_CHECKSUM) ==
Expand All @@ -1192,7 +1187,28 @@ static void DeviceInitPortConf(const DPDKIfaceConfig *iconf,
SCLogConfig("%s: checksum validation enabled (but can be offloaded)", iconf->iface);
}
}
}

static void DeviceInitPortConf(const DPDKIfaceConfig *iconf,
const struct rte_eth_dev_info *dev_info, struct rte_eth_conf *port_conf)
{
DumpRXOffloadCapabilities(dev_info->rx_offload_capa);
*port_conf = (struct rte_eth_conf){
.rxmode = {
.mq_mode = RTE_ETH_MQ_RX_NONE,
.offloads = 0, // turn every offload off to prevent any packet modification
},
.txmode = {
.mq_mode = RTE_ETH_MQ_TX_NONE,
.offloads = 0,
},
};

PortConfSetInterruptMode(iconf, port_conf);

// configure RX offloads
PortConfSetRSSConf(iconf, dev_info, port_conf);
PortConfSetChsumOffload(iconf, dev_info, port_conf);
DeviceSetMTU(port_conf, iconf->mtu);

if (dev_info->tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) {
Expand Down Expand Up @@ -1222,7 +1238,7 @@ static int DeviceConfigureQueues(DPDKIfaceConfig *iconf, const struct rte_eth_de
iconf->mempool_cache_size, 0, mbuf_size, (int)iconf->socket_id);
if (iconf->pkt_mempool == NULL) {
retval = -rte_errno;
SCLogError("%s: rte_pktmbuf_pool_create failed with code %d (mempool: %s) - %s",
SCLogError("%s: rte_pktmbuf_pool_create failed with code %d (mempool: %s): %s",
iconf->iface, rte_errno, mempool_name, rte_strerror(rte_errno));
SCReturnInt(retval);
}
Expand All @@ -1235,34 +1251,39 @@ static int DeviceConfigureQueues(DPDKIfaceConfig *iconf, const struct rte_eth_de
rxq_conf.rx_thresh.wthresh = 0;
rxq_conf.rx_free_thresh = 0;
rxq_conf.rx_drop_en = 0;
SCLogConfig("%s: rx queue setup: queue:%d port:%d rx_desc:%d tx_desc:%d rx: hthresh: %d "
"pthresh %d wthresh %d free_thresh %d drop_en %d offloads %lu",
iconf->iface, queue_id, iconf->port_id, iconf->nb_rx_desc, iconf->nb_tx_desc,
SCLogConfig("%s: setting up RX queue %d: rx_desc: %u offloads: 0x%" PRIx64
" hthresh: %" PRIu8 " pthresh: %" PRIu8 " wthresh: %" PRIu8
" free_thresh: %" PRIu16 " drop_en: %" PRIu8,
iconf->iface, queue_id, iconf->nb_rx_desc, rxq_conf.offloads,
rxq_conf.rx_thresh.hthresh, rxq_conf.rx_thresh.pthresh, rxq_conf.rx_thresh.wthresh,
rxq_conf.rx_free_thresh, rxq_conf.rx_drop_en, rxq_conf.offloads);
rxq_conf.rx_free_thresh, rxq_conf.rx_drop_en);

retval = rte_eth_rx_queue_setup(iconf->port_id, queue_id, iconf->nb_rx_desc,
iconf->socket_id, &rxq_conf, iconf->pkt_mempool);
if (retval < 0) {
rte_mempool_free(iconf->pkt_mempool);
SCLogError(
"%s: rte_eth_rx_queue_setup failed with code %d for device queue %u of port %u",
iconf->iface, retval, queue_id, iconf->port_id);
SCLogError("%s: failed to setup RX queue %u: %s", iconf->iface, queue_id,
rte_strerror(-retval));
SCReturnInt(retval);
}
}

for (uint16_t queue_id = 0; queue_id < iconf->nb_tx_queues; queue_id++) {
txq_conf = dev_info->default_txconf;
txq_conf.offloads = port_conf->txmode.offloads;
SCLogConfig("%s: tx queue setup: queue:%d port:%d", iconf->iface, queue_id, iconf->port_id);
SCLogConfig("%s: setting up TX queue %d: tx_desc: %" PRIu16 " tx: offloads: 0x%" PRIx64
" hthresh: %" PRIu8 " pthresh: %" PRIu8 " wthresh: %" PRIu8
" tx_free_thresh: %" PRIu16 " tx_rs_thresh: %" PRIu16
" txq_deferred_start: %" PRIu8,
iconf->iface, queue_id, iconf->nb_tx_desc, txq_conf.offloads,
txq_conf.tx_thresh.hthresh, txq_conf.tx_thresh.pthresh, txq_conf.tx_thresh.wthresh,
txq_conf.tx_free_thresh, txq_conf.tx_rs_thresh, txq_conf.tx_deferred_start);
retval = rte_eth_tx_queue_setup(
iconf->port_id, queue_id, iconf->nb_tx_desc, iconf->socket_id, &txq_conf);
if (retval < 0) {
rte_mempool_free(iconf->pkt_mempool);
SCLogError(
"%s: rte_eth_tx_queue_setup failed with code %d for device queue %u of port %u",
iconf->iface, retval, queue_id, iconf->port_id);
SCLogError("%s: failed to setup TX queue %u: %s", iconf->iface, queue_id,
rte_strerror(-retval));
SCReturnInt(retval);
}
}
Expand Down Expand Up @@ -1325,21 +1346,23 @@ static int DeviceConfigureIPS(DPDKIfaceConfig *iconf)
if (iconf->out_iface != NULL) {
retval = rte_eth_dev_get_port_by_name(iconf->out_iface, &iconf->out_port_id);
if (retval != 0) {
SCLogError("%s: failed to obtain out iface %s port id (err=%d)", iconf->iface,
iconf->out_iface, retval);
SCLogError("%s: failed to obtain out iface %s port id: %s", iconf->iface,
iconf->out_iface, rte_strerror(-retval));
SCReturnInt(retval);
}

int32_t out_port_socket_id;
retval = DeviceSetSocketID(iconf->port_id, &out_port_socket_id);
if (retval < 0) {
SCLogError("%s: invalid socket id (err=%d)", iconf->out_iface, retval);
SCLogError("%s: invalid socket id: %s", iconf->out_iface, rte_strerror(-retval));
SCReturnInt(retval);
}

if (iconf->socket_id != out_port_socket_id) {
SCLogWarning("%s: out iface %s is not on the same NUMA node", iconf->iface,
iconf->out_iface);
SCLogWarning(
"%s: out iface %s is not on the same NUMA node (%s - NUMA %d, %s - NUMA %d)",
iconf->iface, iconf->out_iface, iconf->iface, iconf->socket_id,
iconf->out_iface, out_port_socket_id);
}

retval = DeviceValidateOutIfaceConfig(iconf);
Expand Down Expand Up @@ -1369,10 +1392,11 @@ static int DeviceConfigureIPS(DPDKIfaceConfig *iconf)
static int32_t DeviceVerifyPostConfigure(
const DPDKIfaceConfig *iconf, const struct rte_eth_dev_info *dev_info)
{
SCEnter();
struct rte_eth_dev_info post_conf_dev_info = { 0 };
int32_t ret = rte_eth_dev_info_get(iconf->port_id, &post_conf_dev_info);
if (ret < 0) {
SCLogError("%s: getting device info failed (err: %s)", iconf->iface, rte_strerror(-ret));
SCLogError("%s: getting device info failed: %s", iconf->iface, rte_strerror(-ret));
SCReturnInt(ret);
}

Expand Down Expand Up @@ -1403,25 +1427,26 @@ static int DeviceConfigure(DPDKIfaceConfig *iconf)
SCEnter();
int32_t retval = rte_eth_dev_get_port_by_name(iconf->iface, &(iconf->port_id));
if (retval < 0) {
SCLogError("%s: getting port id failed (err: %s)", iconf->iface, rte_strerror(-retval));
SCLogError("%s: getting port id failed: %s", iconf->iface, rte_strerror(-retval));
SCReturnInt(retval);
}

if (!rte_eth_dev_is_valid_port(iconf->port_id)) {
SCLogError("%s: specified port %d is invalid", iconf->iface, iconf->port_id);
SCLogError("%s: retrieved port ID \"%d\" is invalid or the device is not attached ",
iconf->iface, iconf->port_id);
SCReturnInt(retval);
}

retval = DeviceSetSocketID(iconf->port_id, &iconf->socket_id);
if (retval < 0) {
SCLogError("%s: invalid socket id (err: %s)", iconf->iface, rte_strerror(-retval));
SCLogError("%s: invalid socket id: %s", iconf->iface, rte_strerror(-retval));
SCReturnInt(retval);
}

struct rte_eth_dev_info dev_info = { 0 };
retval = rte_eth_dev_info_get(iconf->port_id, &dev_info);
if (retval < 0) {
SCLogError("%s: getting device info failed (err: %s)", iconf->iface, rte_strerror(-retval));
SCLogError("%s: getting device info failed: %s", iconf->iface, rte_strerror(-retval));
SCReturnInt(retval);
}

Expand Down Expand Up @@ -1451,21 +1476,25 @@ static int DeviceConfigure(DPDKIfaceConfig *iconf)
retval = rte_eth_dev_configure(
iconf->port_id, iconf->nb_rx_queues, iconf->nb_tx_queues, &port_conf);
if (retval < 0) {
SCLogError("%s: failed to configure the device (port %u, err %s)", iconf->iface,
iconf->port_id, rte_strerror(-retval));
SCLogError("%s: failed to configure the device: %s", iconf->iface, rte_strerror(-retval));
SCReturnInt(retval);
}

retval = DeviceVerifyPostConfigure(iconf, &dev_info);
if (retval < 0)
return retval;

uint16_t tmp_nb_rx_desc = iconf->nb_rx_desc;
uint16_t tmp_nb_tx_desc = iconf->nb_tx_desc;
retval = rte_eth_dev_adjust_nb_rx_tx_desc(
iconf->port_id, &iconf->nb_rx_desc, &iconf->nb_tx_desc);
if (retval != 0) {
SCLogError("%s: failed to adjust device queue descriptors (port %u, err %d)", iconf->iface,
iconf->port_id, retval);
SCLogError("%s: failed to adjust device queue descriptors: %s", iconf->iface,
rte_strerror(-retval));
SCReturnInt(retval);
} else if (tmp_nb_rx_desc != iconf->nb_rx_desc || tmp_nb_tx_desc != iconf->nb_tx_desc) {
SCLogWarning("%s: device queue descriptors adjusted (RX: from %u to %u, TX: from %u to %u)",
iconf->iface, tmp_nb_rx_desc, iconf->nb_rx_desc, tmp_nb_tx_desc, iconf->nb_tx_desc);
}

retval = iconf->flags & DPDK_MULTICAST ? rte_eth_allmulticast_enable(iconf->port_id)
Expand All @@ -1475,17 +1504,15 @@ static int DeviceConfigure(DPDKIfaceConfig *iconf)
// when multicast is enabled but set to disable or vice versa
if ((retval == 1 && !(iconf->flags & DPDK_MULTICAST)) ||
(retval == 0 && (iconf->flags & DPDK_MULTICAST))) {
SCLogError("%s: Allmulticast setting of port (%" PRIu16
") can not be configured. Set it to %s",
iconf->iface, iconf->port_id, retval == 1 ? "true" : "false");
SCLogWarning("%s: cannot configure allmulticast, the port is %sin allmulticast mode",
iconf->iface, retval == 1 ? "" : "not ");
} else if (retval < 0) {
SCLogError("%s: failed to get multicast mode (port %u, err %d)", iconf->iface,
iconf->port_id, retval);
SCLogError("%s: failed to get multicast mode: %s", iconf->iface, rte_strerror(-retval));
SCReturnInt(retval);
}
} else if (retval < 0) {
SCLogError("%s: error when changing multicast setting (port %u err %d)", iconf->iface,
iconf->port_id, retval);
SCLogError("%s: error when changing multicast setting: %s", iconf->iface,
rte_strerror(-retval));
SCReturnInt(retval);
}

Expand All @@ -1495,37 +1522,35 @@ static int DeviceConfigure(DPDKIfaceConfig *iconf)
retval = rte_eth_promiscuous_get(iconf->port_id);
if ((retval == 1 && !(iconf->flags & DPDK_PROMISC)) ||
(retval == 0 && (iconf->flags & DPDK_PROMISC))) {
SCLogError("%s: promiscuous setting of port (%" PRIu16
") can not be configured. Set it to %s",
iconf->iface, iconf->port_id, retval == 1 ? "true" : "false");
SCLogError("%s: cannot configure promiscuous mode, the port is in %spromiscuous mode",
iconf->iface, retval == 1 ? "" : "non-");
SCReturnInt(TM_ECODE_FAILED);
} else if (retval < 0) {
SCLogError("%s: failed to get promiscuous mode (port %u, err=%d)", iconf->iface,
iconf->port_id, retval);
SCLogError(
"%s: failed to get promiscuous mode: %s", iconf->iface, rte_strerror(-retval));
SCReturnInt(retval);
}
} else if (retval < 0) {
SCLogError("%s: error when changing promiscuous setting (port %u, err %d)", iconf->iface,
iconf->port_id, retval);
SCLogError("%s: error when changing promiscuous setting: %s", iconf->iface,
rte_strerror(-retval));
SCReturnInt(TM_ECODE_FAILED);
}

// set maximum transmission unit
SCLogConfig("%s: setting MTU to %d", iconf->iface, iconf->mtu);
retval = rte_eth_dev_set_mtu(iconf->port_id, iconf->mtu);
if (retval == -ENOTSUP) {
SCLogWarning("%s: changing MTU on port %u is not supported, ignoring the setting",
iconf->iface, iconf->port_id);
// if it is not possible to set the MTU, retrieve it
retval = rte_eth_dev_get_mtu(iconf->port_id, &iconf->mtu);
if (retval < 0) {
SCLogError("%s: failed to retrieve MTU (port %u, err %d)", iconf->iface, iconf->port_id,
retval);
SCLogError("%s: failed to retrieve MTU: %s", iconf->iface, rte_strerror(-retval));
SCReturnInt(retval);
}
SCLogWarning(
"%s: changing MTU is not supported, current MTU: %u", iconf->iface, iconf->mtu);
} else if (retval < 0) {
SCLogError("%s: failed to set MTU to %u (port %u, err %d)", iconf->iface, iconf->mtu,
iconf->port_id, retval);
SCLogError(
"%s: failed to set MTU to %u: %s", iconf->iface, iconf->mtu, rte_strerror(-retval));
SCReturnInt(retval);
}

Expand Down Expand Up @@ -1559,7 +1584,7 @@ static void *ParseDpdkConfigAndConfigureDevice(const char *iface)
if (retval < 0) { // handles both configure attempts
iconf->DerefFunc(iconf);
if (rte_eal_cleanup() != 0)
FatalError("EAL cleanup failed: %s", strerror(-retval));
FatalError("EAL cleanup failed: %s", rte_strerror(-retval));

if (retval == -ENOMEM) {
FatalError("%s: memory allocation failed - consider"
Expand Down