Skip to content

Addressing security concerns in Linux Multitenancy #387

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Aug 15, 2019
Merged

Addressing security concerns in Linux Multitenancy #387

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ba0f6ae
Block IP Sppofing in linux multitenacy by verifying the source IP bas…
tamilmani1989 Aug 7, 2019
a39b57e
removed tag while adding portto ovs bridge
tamilmani1989 Aug 7, 2019
2da96db
addressed review comments
tamilmani1989 Aug 14, 2019
71c2047
addressed review comments
tamilmani1989 Aug 15, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 13 additions & 10 deletions network/ovs_endpointclient_linux.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ func (client *OVSEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error {
}

log.Printf("[ovs] Get ovs port for interface %v.", client.hostVethName)
containerPort, err := ovsctl.GetOVSPortNumber(client.hostVethName)
containerOVSPort, err := ovsctl.GetOVSPortNumber(client.hostVethName)
if err != nil {
log.Printf("[ovs] Get ofport failed with error %v", err)
return err
Expand All @@ -103,22 +103,25 @@ func (client *OVSEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error {
return err
}

// IP SNAT Rule
log.Printf("[ovs] Adding IP SNAT rule for egress traffic on %v.", containerPort)
if err := ovsctl.AddIpSnatRule(client.bridgeName, containerPort, client.hostPrimaryMac, ""); err != nil {
return err
}

for _, ipAddr := range epInfo.IPAddresses {
// Add Arp Reply Rules
// Set Vlan id on arp request packet and forward it to table 1
if err := ovsctl.AddFakeArpReply(client.bridgeName, ipAddr.IP); err != nil {
return err
}

// Add IP DNAT rule based on dst ip and vlanid
log.Printf("[ovs] Adding MAC DNAT rule for IP address %v on %v.", ipAddr.IP.String(), hostPort)
if err := ovsctl.AddMacDnatRule(client.bridgeName, hostPort, ipAddr.IP, client.containerMac, client.vlanID); err != nil {
// IP SNAT Rule - Change src mac to VM Mac for packets coming from container host veth port.
// This rule also checks if packets coming from right source ip based on the ovs port to prevent ip spoofing.
// Otherwise it drops the packet.
log.Printf("[ovs] Adding IP SNAT rule for egress traffic on %v.", containerOVSPort)
if err := ovsctl.AddIpSnatRule(client.bridgeName, ipAddr.IP, client.vlanID, containerOVSPort, client.hostPrimaryMac, hostPort); err != nil {
return err
}

// Add IP DNAT rule based on dst ip and vlanid - This rule changes the destination mac to corresponding container mac based on the ip and
// forwards the packet to corresponding container hostveth port
log.Printf("[ovs] Adding MAC DNAT rule for IP address %v on hostport %v, containerport: %v", ipAddr.IP.String(), hostPort, containerOVSPort)
if err := ovsctl.AddMacDnatRule(client.bridgeName, hostPort, ipAddr.IP, client.containerMac, client.vlanID, containerOVSPort); err != nil {
return err
}
}
Expand Down
6 changes: 4 additions & 2 deletions network/ovsinfravnet/infravnet.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,12 +65,14 @@ func (client *OVSInfraVnetClient) CreateInfraVnetRules(
return err
}

if err := ovsctl.AddIpSnatRule(bridgeName, infraContainerPort, hostPrimaryMac, hostPort); err != nil {
// 0 signifies not to add vlan tag to this traffic
if err := ovsctl.AddIpSnatRule(bridgeName, infraIP.IP, 0, infraContainerPort, hostPrimaryMac, hostPort); err != nil {
log.Printf("[ovs] AddIpSnatRule failed with error %v", err)
return err
}

if err := ovsctl.AddMacDnatRule(bridgeName, hostPort, infraIP.IP, client.containerInfraMac, 0); err != nil {
// 0 signifies not to match traffic based on vlan tag
if err := ovsctl.AddMacDnatRule(bridgeName, hostPort, infraIP.IP, client.containerInfraMac, 0, infraContainerPort); err != nil {
log.Printf("[ovs] AddMacDnatRule failed with error %v", err)
return err
}
Expand Down
62 changes: 38 additions & 24 deletions ovsctl/ovsctl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,13 @@ const (
defaultMacForArpResponse = "12:34:56:78:9a:bc"
)

// Open flow rule priorities. Higher the number higher the priority
const (
Copy link
Contributor

@ninzavivek ninzavivek Aug 15, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

consider adding priority to the name and switch to enum?

low = 10
mid = 15
high = 20
)

func CreateOVSBridge(bridgeName string) error {
log.Printf("[ovs] Creating OVS Bridge %v", bridgeName)

Expand Down Expand Up @@ -41,13 +48,7 @@ func DeleteOVSBridge(bridgeName string) error {
}

func AddPortOnOVSBridge(hostIfName string, bridgeName string, vlanID int) error {
cmd := ""

if vlanID == 0 {
cmd = fmt.Sprintf("ovs-vsctl add-port %s %s", bridgeName, hostIfName)
} else {
cmd = fmt.Sprintf("ovs-vsctl add-port %s %s tag=%d", bridgeName, hostIfName, vlanID)
}
cmd := fmt.Sprintf("ovs-vsctl add-port %s %s", bridgeName, hostIfName)
_, err := platform.ExecuteCommand(cmd)
if err != nil {
log.Printf("[ovs] Error while setting OVS as master to primary interface %v", err)
Expand All @@ -69,7 +70,7 @@ func GetOVSPortNumber(interfaceName string) (string, error) {
}

func AddVMIpAcceptRule(bridgeName string, primaryIP string, mac string) error {
cmd := fmt.Sprintf("ovs-ofctl add-flow %s ip,nw_dst=%s,dl_dst=%s,priority=20,actions=normal", bridgeName, primaryIP, mac)
cmd := fmt.Sprintf("ovs-ofctl add-flow %s ip,nw_dst=%s,dl_dst=%s,priority=%d,actions=normal", bridgeName, primaryIP, mac, high)
_, err := platform.ExecuteCommand(cmd)
if err != nil {
log.Printf("[ovs] Adding SNAT rule failed with error %v", err)
Expand All @@ -80,8 +81,8 @@ func AddVMIpAcceptRule(bridgeName string, primaryIP string, mac string) error {
}

func AddArpSnatRule(bridgeName string, mac string, macHex string, ofport string) error {
cmd := fmt.Sprintf(`ovs-ofctl add-flow %v table=1,priority=10,arp,arp_op=1,actions='mod_dl_src:%s,
load:0x%s->NXM_NX_ARP_SHA[],output:%s'`, bridgeName, mac, macHex, ofport)
cmd := fmt.Sprintf(`ovs-ofctl add-flow %v table=1,priority=%d,arp,arp_op=1,actions='mod_dl_src:%s,
load:0x%s->NXM_NX_ARP_SHA[],output:%s'`, bridgeName, low, mac, macHex, ofport)
_, err := platform.ExecuteCommand(cmd)
if err != nil {
log.Printf("[ovs] Adding ARP SNAT rule failed with error %v", err)
Expand All @@ -91,21 +92,32 @@ func AddArpSnatRule(bridgeName string, mac string, macHex string, ofport string)
return nil
}

func AddIpSnatRule(bridgeName string, port string, mac string, outport string) error {
// IP SNAT Rule - Change src mac to VM Mac for packets coming from container host veth port.
func AddIpSnatRule(bridgeName string, ip net.IP, vlanID int, port string, mac string, outport string) error {
var cmd string
if outport == "" {
outport = "normal"
}

cmd := fmt.Sprintf("ovs-ofctl add-flow %v priority=20,ip,in_port=%s,vlan_tci=0,actions=mod_dl_src:%s,strip_vlan,%v",
bridgeName, port, mac, outport)
commonPrefix := fmt.Sprintf("ovs-ofctl add-flow %v priority=%d,ip,nw_src=%s,in_port=%s,vlan_tci=0,actions=mod_dl_src:%s", bridgeName, high, ip.String(), port, mac)

// This rule also checks if packets coming from right source ip based on the ovs port to prevent ip spoofing.
// Otherwise it drops the packet.
if vlanID != 0 {
cmd = fmt.Sprintf("%s,mod_vlan_vid:%v,%v", commonPrefix, vlanID, outport)
} else {
cmd = fmt.Sprintf("%s,strip_vlan,%v", commonPrefix, outport)
}

_, err := platform.ExecuteCommand(cmd)
if err != nil {
log.Printf("[ovs] Adding IP SNAT rule failed with error %v", err)
return err
}

cmd = fmt.Sprintf("ovs-ofctl add-flow %v priority=10,ip,in_port=%s,actions=drop",
bridgeName, port)
// Drop other packets which doesn't satisfy above condition
cmd = fmt.Sprintf("ovs-ofctl add-flow %v priority=%d,ip,in_port=%s,actions=drop",
bridgeName, low, port)
_, err = platform.ExecuteCommand(cmd)
if err != nil {
log.Printf("[ovs] Dropping vlantag packet rule failed with error %v", err)
Expand Down Expand Up @@ -134,11 +146,11 @@ func AddFakeArpReply(bridgeName string, ip net.IP) error {
ipAddrInt := common.IpToInt(ip)

log.Printf("[ovs] Adding ARP reply rule for IP address %v ", ip.String())
cmd := fmt.Sprintf(`ovs-ofctl add-flow %s arp,arp_op=1,priority=20,actions='load:0x2->NXM_OF_ARP_OP[],
cmd := fmt.Sprintf(`ovs-ofctl add-flow %s arp,arp_op=1,priority=%d,actions='load:0x2->NXM_OF_ARP_OP[],
move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:%s,
move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],move:NXM_OF_ARP_TPA[]->NXM_OF_ARP_SPA[],
load:0x%s->NXM_NX_ARP_SHA[],load:0x%x->NXM_OF_ARP_TPA[],IN_PORT'`,
bridgeName, defaultMacForArpResponse, macAddrHex, ipAddrInt)
bridgeName, high, defaultMacForArpResponse, macAddrHex, ipAddrInt)
_, err := platform.ExecuteCommand(cmd)
if err != nil {
log.Printf("[ovs] Adding ARP reply rule failed with error %v", err)
Expand All @@ -163,11 +175,11 @@ func AddArpReplyRule(bridgeName string, port string, ip net.IP, mac string, vlan

// If arp fields matches, set arp reply rule for the request
log.Printf("[ovs] Adding ARP reply rule for IP address %v and vlanid %v.", ip, vlanid)
cmd = fmt.Sprintf(`ovs-ofctl add-flow %s table=1,arp,arp_tpa=%s,dl_vlan=%v,arp_op=1,priority=20,actions='load:0x2->NXM_OF_ARP_OP[],
cmd = fmt.Sprintf(`ovs-ofctl add-flow %s table=1,arp,arp_tpa=%s,dl_vlan=%v,arp_op=1,priority=%d,actions='load:0x2->NXM_OF_ARP_OP[],
move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:%s,
move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[],
load:0x%s->NXM_NX_ARP_SHA[],load:0x%x->NXM_OF_ARP_SPA[],strip_vlan,IN_PORT'`,
bridgeName, ip.String(), vlanid, mac, macAddrHex, ipAddrInt)
bridgeName, ip.String(), vlanid, high, mac, macAddrHex, ipAddrInt)
_, err = platform.ExecuteCommand(cmd)
if err != nil {
log.Printf("[ovs] Adding ARP reply rule failed with error %v", err)
Expand All @@ -177,15 +189,17 @@ func AddArpReplyRule(bridgeName string, port string, ip net.IP, mac string, vlan
return nil
}

func AddMacDnatRule(bridgeName string, port string, ip net.IP, mac string, vlanid int) error {
// Add MAC DNAT rule based on dst ip and vlanid
func AddMacDnatRule(bridgeName string, port string, ip net.IP, mac string, vlanid int, containerPort string) error {
var cmd string
// This rule changes the destination mac to speciifed mac based on the ip and vlanid.
// and forwards the packet to corresponding container hostveth port

commonPrefix := fmt.Sprintf("ovs-ofctl add-flow %s ip,nw_dst=%s,in_port=%s", bridgeName, ip.String(), port)
if vlanid != 0 {
cmd = fmt.Sprintf("ovs-ofctl add-flow %s ip,nw_dst=%s,dl_vlan=%v,in_port=%s,actions=mod_dl_dst:%s,normal",
bridgeName, ip.String(), vlanid, port, mac)
cmd = fmt.Sprintf("%s,dl_vlan=%v,actions=mod_dl_dst:%s,strip_vlan,%s", commonPrefix, vlanid, mac, containerPort)
} else {
cmd = fmt.Sprintf("ovs-ofctl add-flow %s ip,nw_dst=%s,in_port=%s,actions=mod_dl_dst:%s,normal",
bridgeName, ip.String(), port, mac)
cmd = fmt.Sprintf("%s,actions=mod_dl_dst:%s,strip_vlan,%s", commonPrefix, mac, containerPort)
}
_, err := platform.ExecuteCommand(cmd)
if err != nil {
Expand Down