Skip to content

Commit

Permalink
Endpoint for host
Browse files Browse the repository at this point in the history
This commit adds a special Cilium endpoint on each node to represent the
node itself. It takes the reserved security ID 1. The special endpoint is
created by endpoint.CreateLocalNodeEndpoint on daemon startup if that
endpoint wasn't already restored.

When the datapath for the special endpoint is regenerated, it loads
bpf_host.o instead of bpf_lxc.o. bpf_host enforces policies for both
traffic to/from pods and traffic to/from the outside world. It follows
the same template generation path as traditional bpf_lxc files.

ENABLE_EXTRA_HOST_DEV was only defined in init.sh for the from_host path.
It is now defined from the Go side for all paths, and we check whether
from_host is true in the C code instead.

Signed-off-by: Paul Chaignon <paul@cilium.io>
  • Loading branch information
pchaigno authored and aanm committed May 14, 2020
1 parent b211e9e commit a695f53
Show file tree
Hide file tree
Showing 35 changed files with 582 additions and 145 deletions.
37 changes: 23 additions & 14 deletions bpf/Makefile
Expand Up @@ -99,18 +99,18 @@ bpf_overlay.o: bpf_overlay.ll
@$(ECHO_CC)
$(QUIET) ${LLC} ${LLC_FLAGS} -filetype=obj -o $@ $(patsubst %.o,%.ll,$@)

NETDEV_OPTIONS = $(LB_OPTIONS) \
-DENABLE_IPV4:-DENABLE_IPV6:-DLB_L3:-DLB_L4:-DENABLE_HOST_SERVICES_UDP:-DENABLE_HOST_SERVICES_TCP:-DENABLE_EXTERNAL_IP:-DENABLE_DSR \
-DENABLE_IPV4:-DENABLE_IPV6:-DLB_L3:-DLB_L4:-DENABLE_IPSEC:-DENABLE_HOST_SERVICES_UDP:-DENABLE_HOST_SERVICES_TCP:-DENABLE_EXTERNAL_IP:-DENABLE_DSR \
-DENABLE_IPV4:-DENABLE_IPV6:-DLB_L3:-DLB_L4:-DENABLE_HOST_SERVICES_UDP:-DENABLE_HOST_SERVICES_TCP:-DENABLE_NODEPORT:-DENABLE_EXTERNAL_IP:-DENABLE_HOSTPORT \
-DENABLE_IPV4:-DENABLE_IPV6:-DLB_L3:-DLB_L4:-DENABLE_HOST_SERVICES_UDP:-DENABLE_HOST_SERVICES_TCP:-DENABLE_NODEPORT:-DENABLE_EXTERNAL_IP:-DENABLE_MASQUERADE \
-DENABLE_IPV4:-DENABLE_IPV6:-DLB_L3:-DLB_L4:-DENABLE_IPSEC:-DENABLE_HOST_SERVICES_UDP:-DENABLE_HOST_SERVICES_TCP:-DENABLE_NODEPORT:-DENABLE_EXTERNAL_IP:-DENABLE_MASQUERADE \
-DENABLE_IPV4:-DENABLE_IPV6:-DLB_L3:-DLB_L4:-DENABLE_HOST_SERVICES_UDP:-DENABLE_HOST_SERVICES_TCP:-DENABLE_NODEPORT:-DENABLE_EXTERNAL_IP:-DENABLE_DSR \
-DENABLE_IPV4:-DENABLE_IPV6:-DLB_L3:-DLB_L4:-DENABLE_HOST_SERVICES_UDP:-DENABLE_HOST_SERVICES_TCP:-DENABLE_NODEPORT:-DENABLE_EXTERNAL_IP:-DENABLE_DSR:-DENABLE_DSR_HYBRID \
-DENABLE_IPV4:-DENABLE_IPV6:-DLB_L3:-DLB_L4:-DENABLE_HOST_SERVICES_UDP:-DENABLE_HOST_SERVICES_TCP:-DENABLE_NODEPORT:-DENABLE_NODEPORT_ACCELERATION:-DENABLE_EXTERNAL_IP:-DENABLE_DSR:-DENABLE_DSR_HYBRID:-DENABLE_PREFILTER \
-DENABLE_IPV4:-DENABLE_IPV6:-DLB_L3:-DLB_L4:-DENABLE_HOST_SERVICES_UDP:-DENABLE_HOST_SERVICES_TCP:-DENABLE_NODEPORT:-DENABLE_NODEPORT_ACCELERATION:-DENABLE_EXTERNAL_IP:-DENABLE_DSR:-DENABLE_DSR_HYBRID:-DENABLE_PREFILTER:-DENABLE_SESSION_AFFINITY \

MAX_NETDEV_OPTIONS = $(MAX_LB_OPTIONS)
NETDEV_OPTIONS = $(LXC_OPTIONS) \
-DENABLE_IPV4:-DENABLE_IPV6:-DENABLE_HOST_SERVICES_UDP:-DENABLE_HOST_SERVICES_TCP:-DENABLE_EXTERNAL_IP:-DENABLE_DSR \
-DENABLE_IPV4:-DENABLE_IPV6:-DENABLE_IPSEC:-DENABLE_HOST_SERVICES_UDP:-DENABLE_HOST_SERVICES_TCP:-DENABLE_EXTERNAL_IP:-DENABLE_DSR \
-DENABLE_IPV4:-DENABLE_IPV6:-DENABLE_HOST_SERVICES_UDP:-DENABLE_HOST_SERVICES_TCP:-DENABLE_NODEPORT:-DENABLE_EXTERNAL_IP:-DENABLE_HOSTPORT \
-DENABLE_IPV4:-DENABLE_IPV6:-DENABLE_HOST_SERVICES_UDP:-DENABLE_HOST_SERVICES_TCP:-DENABLE_NODEPORT:-DENABLE_EXTERNAL_IP:-DENABLE_MASQUERADE \
-DENABLE_IPV4:-DENABLE_IPV6:-DENABLE_IPSEC:-DENABLE_HOST_SERVICES_UDP:-DENABLE_HOST_SERVICES_TCP:-DENABLE_NODEPORT:-DENABLE_EXTERNAL_IP:-DENABLE_MASQUERADE \
-DENABLE_IPV4:-DENABLE_IPV6:-DENABLE_HOST_SERVICES_UDP:-DENABLE_HOST_SERVICES_TCP:-DENABLE_NODEPORT:-DENABLE_EXTERNAL_IP:-DENABLE_DSR \
-DENABLE_IPV4:-DENABLE_IPV6:-DENABLE_HOST_SERVICES_UDP:-DENABLE_HOST_SERVICES_TCP:-DENABLE_NODEPORT:-DENABLE_EXTERNAL_IP:-DENABLE_DSR:-DENABLE_DSR_HYBRID \
-DENABLE_IPV4:-DENABLE_IPV6:-DENABLE_HOST_SERVICES_UDP:-DENABLE_HOST_SERVICES_TCP:-DENABLE_NODEPORT:-DENABLE_NODEPORT_ACCELERATION:-DENABLE_EXTERNAL_IP:-DENABLE_DSR:-DENABLE_DSR_HYBRID:-DENABLE_PREFILTER \
-DENABLE_IPV4:-DENABLE_IPV6:-DENABLE_HOST_SERVICES_UDP:-DENABLE_HOST_SERVICES_TCP:-DENABLE_NODEPORT:-DENABLE_NODEPORT_ACCELERATION:-DENABLE_EXTERNAL_IP:-DENABLE_DSR:-DENABLE_DSR_HYBRID:-DENABLE_PREFILTER:-DENABLE_SESSION_AFFINITY

MAX_NETDEV_OPTIONS = $(MAX_LXC_OPTIONS)

bpf_host.ll: bpf_host.c $(LIB)
$(QUIET) set -e; \
Expand All @@ -127,16 +127,25 @@ bpf_host.o: bpf_host.ll
@$(ECHO_CC)
$(QUIET) ${LLC} ${LLC_FLAGS} -filetype=obj -o $@ $(patsubst %.o,%.ll,$@)

XDP_OPTIONS = $(LB_OPTIONS) \
-DENABLE_IPV4:-DENABLE_IPV6:-DLB_L3:-DLB_L4:-DENABLE_HOST_SERVICES_UDP:-DENABLE_HOST_SERVICES_TCP:-DENABLE_EXTERNAL_IP:-DENABLE_DSR:-DFROM_HOST \
-DENABLE_IPV4:-DENABLE_IPV6:-DLB_L3:-DLB_L4:-DENABLE_IPSEC:-DENABLE_HOST_SERVICES_UDP:-DENABLE_HOST_SERVICES_TCP:-DENABLE_EXTERNAL_IP:-DENABLE_DSR:-DFROM_HOST \
-DENABLE_IPV4:-DENABLE_IPV6:-DLB_L3:-DLB_L4:-DENABLE_HOST_SERVICES_UDP:-DENABLE_HOST_SERVICES_TCP:-DENABLE_NODEPORT:-DENABLE_EXTERNAL_IP:-DENABLE_HOSTPORT \
-DENABLE_IPV4:-DENABLE_IPV6:-DLB_L3:-DLB_L4:-DENABLE_HOST_SERVICES_UDP:-DENABLE_HOST_SERVICES_TCP:-DENABLE_NODEPORT:-DENABLE_EXTERNAL_IP:-DENABLE_MASQUERADE \
-DENABLE_IPV4:-DENABLE_IPV6:-DLB_L3:-DLB_L4:-DENABLE_IPSEC:-DENABLE_HOST_SERVICES_UDP:-DENABLE_HOST_SERVICES_TCP:-DENABLE_NODEPORT:-DENABLE_EXTERNAL_IP:-DENABLE_MASQUERADE \
-DENABLE_IPV4:-DENABLE_IPV6:-DLB_L3:-DLB_L4:-DENABLE_HOST_SERVICES_UDP:-DENABLE_HOST_SERVICES_TCP:-DENABLE_NODEPORT:-DENABLE_EXTERNAL_IP:-DENABLE_DSR \
-DENABLE_IPV4:-DENABLE_IPV6:-DLB_L3:-DLB_L4:-DENABLE_HOST_SERVICES_UDP:-DENABLE_HOST_SERVICES_TCP:-DENABLE_NODEPORT:-DENABLE_EXTERNAL_IP:-DENABLE_DSR:-DENABLE_DSR_HYBRID

bpf_xdp.ll: bpf_xdp.c $(LIB)
$(QUIET) set -e; \
if [ $(BUILD_PERMUTATIONS) != "" ]; then \
$(foreach OPTS,$(NETDEV_OPTIONS), \
$(foreach OPTS,$(XDP_OPTIONS), \
$(ECHO_CC) " [$(subst :,$(space),$(OPTS))]"; \
${CLANG} $(subst :,$(space),$(OPTS)) ${CLANG_FLAGS} -c $< -o $@; \
${LLC} ${LLC_FLAGS} -o /dev/null $@; ) \
fi
@$(ECHO_CC)
$(QUIET) ${CLANG} ${MAX_NETDEV_OPTIONS} ${CLANG_FLAGS} -c $< -o $@
$(QUIET) ${CLANG} ${MAX_LB_OPTIONS} ${CLANG_FLAGS} -c $< -o $@

bpf_xdp.o: bpf_xdp.ll
@$(ECHO_CC)
Expand Down
10 changes: 8 additions & 2 deletions bpf/bpf_host.c
Expand Up @@ -5,7 +5,13 @@
#include <bpf/api.h>

#include <node_config.h>
#include <netdev_config.h>
#ifdef POD_ENDPOINT
# include <netdev_config.h>
#else
# include <ep_config.h>
# define EVENT_SOURCE HOST_EP_ID
#endif


/* These are configuartion options which have a default value in their
* respective header files and must thus be defined beforehand:
Expand Down Expand Up @@ -304,7 +310,7 @@ resolve_srcid_ipv4(struct __ctx_buff *ctx, __u32 srcid_from_proxy,
* source as HOST_ID.
*/
#ifndef ENABLE_EXTRA_HOST_DEV
if (sec_label != HOST_ID)
if (from_host && sec_label != HOST_ID)
#endif
srcid_from_ipcache = sec_label;
}
Expand Down
2 changes: 1 addition & 1 deletion bpf/bpf_lxc.c
Expand Up @@ -5,7 +5,7 @@
#include <bpf/api.h>

#include <node_config.h>
#include <lxc_config.h>
#include <ep_config.h>

#include <linux/icmpv6.h>

Expand Down
6 changes: 6 additions & 0 deletions bpf/lxc_config.h → bpf/ep_config.h
Expand Up @@ -10,8 +10,14 @@
DEFINE_IPV6(LXC_IP, 0xbe, 0xef, 0, 0, 0, 0, 0, 0x1, 0, 0, 0, 0x1, 0x01, 0x65, 0x82, 0xbc);
DEFINE_U32(LXC_IPV4, 0x10203040);
#define LXC_IPV4 fetch_u32(LXC_IPV4)
/*
* Both the LXC_ID and the HOST_EP_ID are defined here to ease compile testing,
* but in the actual header files, only one of them will be present.
*/
DEFINE_U32(LXC_ID, 0x2A);
#define LXC_ID fetch_u32(LXC_ID)
DEFINE_U32(HOST_EP_ID, 0x1092);
#define HOST_EP_ID fetch_u32(HOST_EP_ID)
DEFINE_U32(SECLABEL, 0xfffff);
#define SECLABEL fetch_u32(SECLABEL)
DEFINE_U32(SECLABEL_NB, 0xfffff);
Expand Down
33 changes: 2 additions & 31 deletions bpf/init.sh
Expand Up @@ -579,29 +579,6 @@ if [ "$MODE" = "direct" ] || [ "$MODE" = "ipvlan" ] || [ "$MODE" = "routed" ] ||
fi
fi

for NATIVE_DEV in ${NATIVE_DEVS//;/ }; do
LOCAL_COPTS="${NP_COPTS} -DSECLABEL=${ID_WORLD}"
NATIVE_DEV_IDX=$(cat /sys/class/net/${NATIVE_DEV}/ifindex)
CALLS_MAP=cilium_calls_netdev_${NATIVE_DEV_IDX}

if [ "$NODE_PORT" = "true" ]; then
LOCAL_COPTS="${LOCAL_COPTS} -DLB_L3 -DLB_L4 -DDISABLE_LOOPBACK_LB -DNATIVE_DEV_IFINDEX=${NATIVE_DEV_IDX}"
if [ "$IP4_HOST" != "<nil>" ]; then
LOCAL_COPTS="${LOCAL_COPTS} -DIPV4_NODEPORT=${v4_addrs[$NATIVE_DEV]}"
fi
if [ "$IP6_HOST" != "<nil>" ]; then
LOCAL_COPTS="${LOCAL_COPTS} -DIPV6_NODEPORT={.addr={${v6_addrs[$NATIVE_DEV]}}}"
fi
fi

bpf_load $NATIVE_DEV "$LOCAL_COPTS" "ingress" bpf_host.c bpf_host.o "from-netdev" $CALLS_MAP
if [ "$NODE_PORT" = "true" ]; then
bpf_load $NATIVE_DEV "$LOCAL_COPTS" "egress" bpf_host.c bpf_host.o "to-netdev" $CALLS_MAP
else
bpf_unload $NATIVE_DEV "egress"
fi
done

echo "$NATIVE_DEVS" > $RUNDIR/device.state
fi
else
Expand Down Expand Up @@ -688,15 +665,9 @@ else
bpf_clear_cgroups $CGROUP_ROOT recvmsg6
fi

# bpf_host.o requires to see an updated node_config.h which includes ENCAP_IFINDEX
COPTS="-DPOD_ENDPOINT=1 -DSECLABEL=${ID_WORLD}"
CALLS_MAP="cilium_calls_netdev_ns_${ID_HOST}"
COPTS="-DSECLABEL=${ID_HOST}"
if [ "$MODE" == "ipvlan" ]; then
COPTS+=" -DENABLE_EXTRA_HOST_DEV"
fi
bpf_load $HOST_DEV1 "$COPTS" "egress" bpf_host.c bpf_host.o from-host $CALLS_MAP
bpf_load $HOST_DEV1 "" "ingress" bpf_host.c bpf_host.o to-host $CALLS_MAP
bpf_load $HOST_DEV2 "" "ingress" bpf_host.c bpf_host.o to-host $CALLS_MAP
bpf_load $HOST_DEV2 "$COPTS" "ingress" bpf_host.c bpf_host.o to-host $CALLS_MAP
if [ "$IPSEC" == "true" ]; then
if [ "$ENCRYPT_DEV" != "<nil>" ]; then
bpf_load $ENCRYPT_DEV "" "ingress" bpf_network.c bpf_network.o from-network $CALLS_MAP
Expand Down
23 changes: 9 additions & 14 deletions bpf/lib/nodeport.h
Expand Up @@ -26,17 +26,19 @@
#if defined(ENABLE_IPV4) && !defined(IPV4_NODEPORT)
#define IPV4_NODEPORT 0
#endif
#if defined(ENABLE_IPV6) && !defined(IPV6_NODEPORT)
#define IPV6_NODEPORT { .addr = { 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 } }
#if defined(ENABLE_IPV6) && !defined(IPV6_NODEPORT_V)
DEFINE_IPV6(IPV6_NODEPORT, 0xbe, 0xef, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x1, 0x0, 0x1, 0x0, 0x0);
# define IPV6_NODEPORT_V
#endif
#ifndef DIRECT_ROUTING_DEV_IFINDEX
#define DIRECT_ROUTING_DEV_IFINDEX 0
#endif
#if defined(ENABLE_IPV4) && !defined(IPV4_DIRECT_ROUTING)
#define IPV4_DIRECT_ROUTING 0
#endif
#if defined(ENABLE_IPV6) && !defined(IPV6_DIRECT_ROUTING)
#define IPV6_DIRECT_ROUTING { .addr = { 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 } }
#if defined(ENABLE_IPV6) && !defined(IPV6_DIRECT_ROUTING_V)
DEFINE_IPV6(IPV6_DIRECT_ROUTING, 0xbe, 0xef, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x1, 0x0, 0x1, 0x0, 0x0);
# define IPV6_DIRECT_ROUTING_V
#endif
#endif /* ENABLE_NODEPORT */

Expand Down Expand Up @@ -364,10 +366,8 @@ int tail_nodeport_nat_ipv6(struct __ctx_buff *ctx)
};
void *data, *data_end;
struct ipv6hdr *ip6;
union v6addr tmp = IPV6_DIRECT_ROUTING;


target.addr = tmp;
BPF_V6(target.addr, IPV6_DIRECT_ROUTING);
#ifdef ENCAP_IFINDEX
if (dir == NAT_DIR_EGRESS) {
struct remote_endpoint_info *info;
Expand Down Expand Up @@ -1379,14 +1379,9 @@ static __always_inline int nodeport_nat_fwd(struct __ctx_buff *ctx,
#ifdef ENCAP_IFINDEX
if (encap)
BPF_V6(addr, ROUTER_IP);
else {
#endif
union v6addr tmp = IPV6_NODEPORT;

addr = tmp;
#ifdef ENCAP_IFINDEX
}
else
#endif
BPF_V6(addr, IPV6_NODEPORT);
return nodeport_nat_ipv6_fwd(ctx, &addr);
}
#endif /* ENABLE_IPV6 */
Expand Down
3 changes: 0 additions & 3 deletions bpf/node_config.h
Expand Up @@ -20,7 +20,6 @@ DEFINE_IPV6(ROUTER_IP, 0xbe, 0xef, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0,
#define ENCAP_IFINDEX 1
#define HOST_IFINDEX 1
#define CILIUM_IFINDEX 1
#define NATIVE_DEV_IFINDEX 1
#define NATIVE_DEV_MAC_BY_IFINDEX(_) { .addr = { 0xce, 0x72, 0xa7, 0x03, 0x88, 0x56 } }
DEFINE_IPV6(HOST_IP, 0xbe, 0xef, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xa, 0x0, 0x2, 0xf, 0xff, 0xff);
#define HOST_ID 1
Expand Down Expand Up @@ -121,11 +120,9 @@ DEFINE_IPV6(HOST_IP, 0xbe, 0xef, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xa, 0x
#ifdef ENABLE_NODEPORT
#ifdef ENABLE_IPV4
#define NODEPORT_NEIGH4 test_cilium_neigh4
#define IPV4_NODEPORT 0x10203040
#endif
#ifdef ENABLE_IPV6
#define NODEPORT_NEIGH6 test_cilium_neigh6
DEFINE_IPV6(IPV6_NODEPORT, 0xbe, 0xef, 0, 0, 0, 0, 0, 0x1, 0, 0, 0, 0x1, 0x01, 0x65, 0x82, 0xbc);
#endif
#endif

Expand Down
8 changes: 8 additions & 0 deletions daemon/cmd/daemon_main.go
Expand Up @@ -1291,6 +1291,14 @@ func runDaemon() {
bootstrapStats.k8sInit.End(true)
restoreComplete := d.initRestore(restoredEndpoints)

if !d.endpointManager.HostEndpointExists() {
log.Info("Creating host endpoint")
if err := d.endpointManager.AddHostEndpoint(d.ctx, d, d.l7Proxy, d.identityAllocator,
"Create host endpoint", nodeTypes.GetName()); err != nil {
log.WithError(err).Fatal("Unable to create host endpoint")
}
}

if option.Config.IsFlannelMasterDeviceSet() {
if option.Config.EnableEndpointHealthChecking {
log.Warn("Running Cilium in flannel mode doesn't support endpoint connectivity health checking. Disabling endpoint connectivity health check.")
Expand Down
2 changes: 1 addition & 1 deletion daemon/cmd/policy_test.go
Expand Up @@ -195,7 +195,7 @@ func prepareEndpointDirs() (cleanup func(), err error) {
return func() {}, err
}
return func() {
os.RemoveAll(fmt.Sprintf("%s/lxc_config.h", testEPDir))
os.RemoveAll(fmt.Sprintf("%s/ep_config.h", testEPDir))
time.Sleep(1 * time.Second)
os.RemoveAll(testEPDir)
os.RemoveAll(fmt.Sprintf("%s_backup", testEPDir))
Expand Down
2 changes: 1 addition & 1 deletion pkg/common/const.go
Expand Up @@ -32,7 +32,7 @@ const (
NodeConfigFile = "node_config.h"
// CHeaderFileName is the name of the C header file for BPF programs for a
// particular endpoint.
CHeaderFileName = "lxc_config.h"
CHeaderFileName = "ep_config.h"
// NetdevHeaderFileName is the name of the header file used for bpf_host.c and bpf_overlay.c.
NetdevHeaderFileName = "netdev_config.h"
// PreFilterHeaderFileName is the name of the header file used for bpf_xdp.c.
Expand Down
3 changes: 3 additions & 0 deletions pkg/datapath/config.go
Expand Up @@ -82,6 +82,9 @@ type CompileTimeConfiguration interface {

// GetPolicyVerdictLogFilter returns the PolicyVerdictLogFilter for the endpoint
GetPolicyVerdictLogFilter() uint32

// IsHost returns true if the endpoint is the host endpoint.
IsHost() bool
}

// EndpointConfiguration provides datapath implementations a clean interface
Expand Down
1 change: 1 addition & 0 deletions pkg/datapath/endpoint.go
Expand Up @@ -24,4 +24,5 @@ type Endpoint interface {
Logger(subsystem string) *logrus.Entry
StateDir() string
MapPath() string
IsHost() bool
}
18 changes: 18 additions & 0 deletions pkg/datapath/link/link.go
Expand Up @@ -17,6 +17,8 @@ package link
import (
"fmt"

"github.com/cilium/cilium/pkg/mac"

"github.com/vishvananda/netlink"
)

Expand All @@ -43,3 +45,19 @@ func Rename(curName, newName string) error {

return netlink.LinkSetName(link, newName)
}

func GetHardwareAddr(ifName string) (mac.MAC, error) {
iface, err := netlink.LinkByName(ifName)
if err != nil {
return nil, err
}
return mac.MAC(iface.Attrs().HardwareAddr), nil
}

func GetIfIndex(ifName string) (uint32, error) {
iface, err := netlink.LinkByName(ifName)
if err != nil {
return 0, err
}
return uint32(iface.Attrs().Index), nil
}

0 comments on commit a695f53

Please sign in to comment.