Permalink
Browse files

Merge pull request #3267 from mjstapp/dplane_3

zebra async dataplane: phase 2
  • Loading branch information...
riw777 committed Nov 27, 2018
2 parents 98d8359 + 80776ae commit 4fedcc69476f5e35e1f5c344e6ae13eca922ade6
Showing with 987 additions and 175 deletions.
  1. +51 −14 zebra/kernel_netlink.c
  2. +17 −3 zebra/kernel_socket.c
  3. +9 −3 zebra/main.c
  4. +1 −1 zebra/rt.h
  5. +771 −124 zebra/zebra_dplane.c
  6. +96 −18 zebra/zebra_dplane.h
  7. +34 −5 zebra/zebra_ns.c
  8. +5 −3 zebra/zebra_ns.h
  9. +3 −4 zebra/zebra_rib.c
@@ -396,7 +396,7 @@ static int kernel_read(struct thread *thread)

/*
* Filter out messages from self that occur on listener socket,
* caused by our actions on the command socket
* caused by our actions on the command socket(s)
*
* When we add new Netlink message types we probably
* do not need to add them here as that we are filtering
@@ -407,7 +407,7 @@ static int kernel_read(struct thread *thread)
* so that we only had to write one way to handle incoming
* address add/delete changes.
*/
static void netlink_install_filter(int sock, __u32 pid)
static void netlink_install_filter(int sock, __u32 pid, __u32 dplane_pid)
{
/*
* BPF_JUMP instructions and where you jump to are based upon
@@ -418,7 +418,8 @@ static void netlink_install_filter(int sock, __u32 pid)
struct sock_filter filter[] = {
/*
* Logic:
* if (nlmsg_pid == pid) {
* if (nlmsg_pid == pid ||
* nlmsg_pid == dplane_pid) {
* if (the incoming nlmsg_type ==
* RTM_NEWADDR | RTM_DELADDR)
* keep this message
@@ -435,26 +436,30 @@ static void netlink_install_filter(int sock, __u32 pid)
/*
* 1: Compare to pid
*/
BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, htonl(pid), 0, 4),
BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, htonl(pid), 1, 0),
/*
* 2: Load the nlmsg_type into BPF register
* 2: Compare to dplane pid
*/
BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, htonl(dplane_pid), 0, 4),
/*
* 3: Load the nlmsg_type into BPF register
*/
BPF_STMT(BPF_LD | BPF_ABS | BPF_H,
offsetof(struct nlmsghdr, nlmsg_type)),
/*
* 3: Compare to RTM_NEWADDR
* 4: Compare to RTM_NEWADDR
*/
BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, htons(RTM_NEWADDR), 2, 0),
/*
* 4: Compare to RTM_DELADDR
* 5: Compare to RTM_DELADDR
*/
BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, htons(RTM_DELADDR), 1, 0),
/*
* 5: This is the end state of we want to skip the
* 6: This is the end state of we want to skip the
* message
*/
BPF_STMT(BPF_RET | BPF_K, 0),
/* 6: This is the end state of we want to keep
/* 7: This is the end state of we want to keep
* the message
*/
BPF_STMT(BPF_RET | BPF_K, 0xffff),
@@ -1102,6 +1107,15 @@ void kernel_init(struct zebra_ns *zns)
exit(-1);
}

snprintf(zns->netlink_dplane.name, sizeof(zns->netlink_dplane.name),
"netlink-dp (NS %u)", zns->ns_id);
zns->netlink_dplane.sock = -1;
if (netlink_socket(&zns->netlink_dplane, 0, zns->ns_id) < 0) {
zlog_err("Failure to create %s socket",
zns->netlink_dplane.name);
exit(-1);
}

/*
* SOL_NETLINK is not available on all platforms yet
* apparently. It's in bits/socket.h which I am not
@@ -1110,14 +1124,22 @@ void kernel_init(struct zebra_ns *zns)
#if defined SOL_NETLINK
/*
* Let's tell the kernel that we want to receive extended
* ACKS over our command socket
* ACKS over our command socket(s)
*/
one = 1;
ret = setsockopt(zns->netlink_cmd.sock, SOL_NETLINK, NETLINK_EXT_ACK,
&one, sizeof(one));

if (ret < 0)
zlog_notice("Registration for extended ACK failed : %d %s",
zlog_notice("Registration for extended cmd ACK failed : %d %s",
errno, safe_strerror(errno));

one = 1;
ret = setsockopt(zns->netlink_dplane.sock, SOL_NETLINK, NETLINK_EXT_ACK,
&one, sizeof(one));

if (ret < 0)
zlog_notice("Registration for extended dp ACK failed : %d %s",
errno, safe_strerror(errno));
#endif

@@ -1130,12 +1152,18 @@ void kernel_init(struct zebra_ns *zns)
zlog_err("Can't set %s socket error: %s(%d)",
zns->netlink_cmd.name, safe_strerror(errno), errno);

if (fcntl(zns->netlink_dplane.sock, F_SETFL, O_NONBLOCK) < 0)
zlog_err("Can't set %s socket error: %s(%d)",
zns->netlink_dplane.name, safe_strerror(errno), errno);

/* Set receive buffer size if it's set from command line */
if (nl_rcvbufsize)
netlink_recvbuf(&zns->netlink, nl_rcvbufsize);

netlink_install_filter(zns->netlink.sock,
zns->netlink_cmd.snl.nl_pid);
zns->netlink_cmd.snl.nl_pid,
zns->netlink_dplane.snl.nl_pid);

zns->t_netlink = NULL;

thread_add_read(zebrad.master, kernel_read, zns,
@@ -1144,7 +1172,7 @@ void kernel_init(struct zebra_ns *zns)
rt_netlink_init();
}

void kernel_terminate(struct zebra_ns *zns)
void kernel_terminate(struct zebra_ns *zns, bool complete)
{
THREAD_READ_OFF(zns->t_netlink);

@@ -1157,6 +1185,15 @@ void kernel_terminate(struct zebra_ns *zns)
close(zns->netlink_cmd.sock);
zns->netlink_cmd.sock = -1;
}
}

/* During zebra shutdown, we need to leave the dataplane socket
* around until all work is done.
*/
if (complete) {
if (zns->netlink_dplane.sock >= 0) {
close(zns->netlink_dplane.sock);
zns->netlink_dplane.sock = -1;
}
}
}
#endif /* HAVE_NETLINK */
@@ -278,6 +278,11 @@ static const struct message rtm_flag_str[] = {{RTF_UP, "UP"},
/* Kernel routing update socket. */
int routing_sock = -1;

/* Kernel dataplane routing update socket, used in the dataplane pthread
* context.
*/
int dplane_routing_sock = -1;

/* Yes I'm checking ugly routing socket behavior. */
/* #define DEBUG */

@@ -1136,7 +1141,7 @@ int rtm_write(int message, union sockunion *dest, union sockunion *mask,
char buf[512];
} msg;

if (routing_sock < 0)
if (dplane_routing_sock < 0)
return ZEBRA_ERR_EPERM;

/* Clear and set rt_msghdr values */
@@ -1243,7 +1248,7 @@ int rtm_write(int message, union sockunion *dest, union sockunion *mask,

msg.rtm.rtm_msglen = pnt - (caddr_t)&msg;

ret = write(routing_sock, &msg, msg.rtm.rtm_msglen);
ret = write(dplane_routing_sock, &msg, msg.rtm.rtm_msglen);

if (ret != msg.rtm.rtm_msglen) {
if (errno == EEXIST)
@@ -1390,13 +1395,22 @@ static void routing_socket(struct zebra_ns *zns)
{
frr_elevate_privs(&zserv_privs) {
routing_sock = ns_socket(AF_ROUTE, SOCK_RAW, 0, zns->ns_id);

dplane_routing_sock =
ns_socket(AF_ROUTE, SOCK_RAW, 0, zns->ns_id);
}

if (routing_sock < 0) {
flog_err_sys(EC_LIB_SOCKET, "Can't init kernel routing socket");
return;
}

if (dplane_routing_sock < 0) {
flog_err_sys(EC_LIB_SOCKET,
"Can't init kernel dataplane routing socket");
return;
}

/* XXX: Socket should be NONBLOCK, however as we currently
* discard failed writes, this will lead to inconsistencies.
* For now, socket must be blocking.
@@ -1415,7 +1429,7 @@ void kernel_init(struct zebra_ns *zns)
routing_socket(zns);
}

void kernel_terminate(struct zebra_ns *zns)
void kernel_terminate(struct zebra_ns *zns, bool complete)
{
return;
}
@@ -172,7 +172,7 @@ static void sigint(void)
work_queue_free_and_null(&zebrad.lsp_process_q);
vrf_terminate();

ns_walk_func(zebra_ns_disabled);
ns_walk_func(zebra_ns_early_shutdown);
zebra_ns_notify_close();

access_list_reset();
@@ -196,6 +196,9 @@ int zebra_finalize(struct thread *dummy)
{
zlog_info("Zebra final shutdown");

/* Final shutdown of ns resources */
ns_walk_func(zebra_ns_final_shutdown);

/* Stop dplane thread and finish any cleanup */
zebra_dplane_shutdown();

@@ -390,6 +393,9 @@ int main(int argc, char **argv)
vty_config_lockless();
zebrad.master = frr_init();

/* Initialize pthread library */
frr_pthread_init();

/* Zebra related initialize. */
zebra_router_init();
zserv_init();
@@ -445,8 +451,8 @@ int main(int argc, char **argv)
/* Needed for BSD routing socket. */
pid = getpid();

/* Intialize pthread library */
frr_pthread_init();
/* Start dataplane system */
zebra_dplane_start();

/* Start Zebra API server */
zserv_start(zserv_path);
@@ -86,7 +86,7 @@ extern int kernel_del_neigh(struct interface *ifp, struct ipaddr *ip);
*/
extern void interface_list(struct zebra_ns *zns);
extern void kernel_init(struct zebra_ns *zns);
extern void kernel_terminate(struct zebra_ns *zns);
extern void kernel_terminate(struct zebra_ns *zns, bool complete);
extern void macfdb_read(struct zebra_ns *zns);
extern void macfdb_read_for_bridge(struct zebra_ns *zns, struct interface *ifp,
struct interface *br_if);
Oops, something went wrong.

0 comments on commit 4fedcc6

Please sign in to comment.