diff --git a/include/common.h b/include/common.h index def41b4b4..d8c89aa56 100644 --- a/include/common.h +++ b/include/common.h @@ -107,6 +107,8 @@ enum { extern const char *dpvs_strerror(int err); +int get_numa_nodes(void); + int linux_set_if_mac(const char *ifname, const unsigned char mac[ETH_ALEN]); int linux_hw_mc_add(const char *ifname, const uint8_t hwma[ETH_ALEN]); int linux_hw_mc_del(const char *ifname, const uint8_t hwma[ETH_ALEN]); diff --git a/src/common.c b/src/common.c index 7172bbe28..c2193952f 100644 --- a/src/common.c +++ b/src/common.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -85,6 +86,18 @@ dpvs_state_t dpvs_state_get(void) return g_dpvs_tate; } +int get_numa_nodes(void) +{ + int numa_nodes; + + if (numa_available() < 0) + numa_nodes = 0; + else + numa_nodes = numa_max_node(); + + return (numa_nodes + 1); +} + /* if (num+offset) == 2^n, return true, * otherwise return false and 'lower' is filled with * the closest lower bound value to 'num' */ diff --git a/src/config.mk b/src/config.mk index 5969ad076..d787b1443 100644 --- a/src/config.mk +++ b/src/config.mk @@ -21,7 +21,7 @@ # TODO: use standard way to define compile flags. # -CFLAGS += -D DPVS_MAX_SOCKET=1 +CFLAGS += -D DPVS_MAX_SOCKET=2 CFLAGS += -D DPVS_MAX_LCORE=64 #CFLAGS += -D CONFIG_RECORD_BIG_LOOP diff --git a/src/ipvs/ip_vs_conn.c b/src/ipvs/ip_vs_conn.c index ae7afe665..07ce42010 100644 --- a/src/ipvs/ip_vs_conn.c +++ b/src/ipvs/ip_vs_conn.c @@ -1480,7 +1480,7 @@ int dp_vs_conn_init(void) conn_ctrl_init(); /* connection cache on each NUMA socket */ - for (i = 0; i < DPVS_MAX_SOCKET; i++) { + for (i = 0; i < get_numa_nodes(); i++) { snprintf(poolname, sizeof(poolname), "dp_vs_conn_%d", i); dp_vs_conn_cache[i] = rte_mempool_create(poolname, conn_pool_size, diff --git a/src/ipvs/ip_vs_synproxy.c b/src/ipvs/ip_vs_synproxy.c index 7abe7544a..17dd937fe 100644 --- a/src/ipvs/ip_vs_synproxy.c +++ b/src/ipvs/ip_vs_synproxy.c @@ -124,7 +124,7 @@ int dp_vs_synproxy_init(void) dpvs_timer_sched(&g_minute_timer, &tv, minute_timer_expire, NULL, true); /* allocate NUMA-aware ACK list cache */ - for (i = 0; i < DPVS_MAX_SOCKET; i++) { + for (i = 0; i < get_numa_nodes(); i++) { snprintf(ack_mbufpool_name, sizeof(ack_mbufpool_name), "ack_mbufpool_%d", i); dp_vs_synproxy_ack_mbufpool[i] = rte_mempool_create(ack_mbufpool_name, DP_VS_SYNPROXY_ACK_MBUFPOOL_SIZE, @@ -153,7 +153,7 @@ int dp_vs_synproxy_term(void) int i; dpvs_timer_cancel(&g_minute_timer, true); - for (i = 0; i < DPVS_MAX_SOCKET; i++) + for (i = 0; i < get_numa_nodes(); i++) rte_mempool_free(dp_vs_synproxy_ack_mbufpool[i]); return EDPVS_OK; diff --git a/src/kni.c b/src/kni.c index 66af8f00a..cd382ba8c 100644 --- a/src/kni.c +++ b/src/kni.c @@ -382,7 +382,7 @@ int kni_init(void) int i; char poolname[32]; - for (i = 0; i < DPVS_MAX_SOCKET; i++) { + for (i = 0; i < get_numa_nodes(); i++) { memset(poolname, 0, sizeof(poolname)); snprintf(poolname, sizeof(poolname) - 1, "kni_mbuf_pool_%d", i); diff --git a/src/main.c b/src/main.c index 0c9cfa6d1..165da0e99 100644 --- a/src/main.c +++ b/src/main.c @@ -16,7 +16,6 @@ * */ #define _GNU_SOURCE -#include #include #include #include "pidfile.h" @@ -76,7 +75,7 @@ static int set_all_thread_affinity(void) int main(int argc, char *argv[]) { - int err, nports, numa_nodes; + int err, nports; portid_t pid; struct netif_port *dev; struct timeval tv; @@ -96,14 +95,8 @@ int main(int argc, char *argv[]) gettimeofday(&tv, NULL); srandom(tv.tv_sec ^ tv.tv_usec ^ getpid()); - if (numa_available() < 0) - numa_nodes = 0; - else - numa_nodes = numa_max_node(); - numa_nodes++; - - if (numa_nodes != DPVS_MAX_SOCKET) { - fprintf(stderr, "DPVS_MAX_SOCKET and system numa nodes do not match!\n"); + if (get_numa_nodes() > DPVS_MAX_SOCKET) { + fprintf(stderr, "DPVS_MAX_SOCKET is smaller than system numa nodes!\n"); return -1; } diff --git a/src/netif.c b/src/netif.c index b7fba585c..a1ae76f80 100644 --- a/src/netif.c +++ b/src/netif.c @@ -879,7 +879,7 @@ static inline void netif_pktmbuf_pool_init(void) { int i; char poolname[32]; - for (i = 0; i < DPVS_MAX_SOCKET; i++) { + for (i = 0; i < get_numa_nodes(); i++) { snprintf(poolname, sizeof(poolname), "mbuf_pool_%d", i); pktmbuf_pool[i] = rte_pktmbuf_pool_create(poolname, netif_pktpool_nb_mbuf, netif_pktpool_mbuf_cache, 0, RTE_MBUF_DEFAULT_BUF_SIZE, i); diff --git a/src/tc/tc.c b/src/tc/tc.c index 94cd2c404..45ddc249c 100644 --- a/src/tc/tc.c +++ b/src/tc/tc.c @@ -362,7 +362,7 @@ int tc_init(void) tc_register_cls(&match_cls_ops); /* per-NUMA socket mempools for queued tc_mbuf{} */ - for (s = 0; s < DPVS_MAX_SOCKET; s++) { + for (s = 0; s < get_numa_nodes(); s++) { char plname[64]; snprintf(plname, sizeof(plname), "tc_mbuf_pool_%d", s); diff --git a/tools/dpip/Makefile b/tools/dpip/Makefile index 96ed4bf9a..60f6ac60c 100644 --- a/tools/dpip/Makefile +++ b/tools/dpip/Makefile @@ -27,8 +27,10 @@ CFLAGS += -Wall -Werror -Wstrict-prototypes -Wmissing-prototypes CFLAGS += -I ../../include CFLAGS += -I ../keepalived/keepalived/libipvs-2.6 +LIBS = -lnuma DEFS = -D DPVS_MAX_LCORE=64 -CFLAGS += $(DEFS) + +CFLAGS += $(LIBS) $(DEFS) OBJS = dpip.o utils.o route.o addr.o neigh.o link.o vlan.o \ qsch.o cls.o tunnel.o ../../src/common.o \ diff --git a/tools/ipvsadm/Makefile b/tools/ipvsadm/Makefile index 4862e095a..f4bfbff3d 100644 --- a/tools/ipvsadm/Makefile +++ b/tools/ipvsadm/Makefile @@ -83,7 +83,7 @@ POPT_DEFINE = -DHAVE_POPT endif OBJS = ipvsadm.o config_stream.o dynamic_array.o -LIBS = $(POPT_LIB) +LIBS = $(POPT_LIB) -lnuma DEFINES = -DVERSION=\"$(VERSION)\" -DSCHEDULERS=\"$(SCHEDULERS)\" \ -DPE_LIST=\"$(PE_LIST)\" $(POPT_DEFINE) DEFINES += $(shell if [ ! -f ../ip_vs.h ]; then \ diff --git a/tools/keepalived/keepalived/Makefile.in b/tools/keepalived/keepalived/Makefile.in index e6618b469..bcd2732dd 100644 --- a/tools/keepalived/keepalived/Makefile.in +++ b/tools/keepalived/keepalived/Makefile.in @@ -22,7 +22,7 @@ sysconf_dir = $(sysconfdir)/sysconfig CC = @CC@ STRIP = @STRIP@ -LDFLAGS = @LIBS@ @LDFLAGS@ -ldl +LDFLAGS = @LIBS@ @LDFLAGS@ -ldl -lnuma SUBDIRS = core DPVSDEPS = ../../../src/common.o