From 2734f2ae7813cc27a32886272af67e66f1854a66 Mon Sep 17 00:00:00 2001 From: eduards Date: Thu, 29 Sep 2016 10:52:55 -0700 Subject: [PATCH] Better NetNS parallel access I think GO is doing something fuzzy when lockOSthread is called, and the goroutine sleeps. This should avoid a goroutine to lock the thead and sleep, forcing the routine to always unlock the thread prior going to sleep, allowing other routines to reuse the thread. Tested it with 1-shot 90 container onboard, no issues seen. Signed-off-by: eduards Conflicts: plugin/driver/driver.go Signed-off-by: eduards --- plugin/driver/driver.go | 17 +++++++++------ plugin/driver/network_utils.go | 39 +++++++--------------------------- 2 files changed, 18 insertions(+), 38 deletions(-) diff --git a/plugin/driver/driver.go b/plugin/driver/driver.go index 44b3e00..2aff41a 100644 --- a/plugin/driver/driver.go +++ b/plugin/driver/driver.go @@ -25,6 +25,7 @@ import ( "os/exec" "runtime" "strings" + "time" Log "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/drivers/remote/api" @@ -352,7 +353,7 @@ func (driver *driver) joinEndpoint(w http.ResponseWriter, r *http.Request) { } objectResponse(w, res) - Log.Infof("Join endpoint %s:%s to %s", j.NetworkID, j.EndpointID, j.SandboxKey) + Log.Infof("Join endpoint %s: %s to %s", j.NetworkID, j.EndpointID, j.SandboxKey) if auto_arp { go func(net_ns_path string, pg_ifc_prefix string) { @@ -360,12 +361,14 @@ func (driver *driver) joinEndpoint(w http.ResponseWriter, r *http.Request) { // Disallow this goroutine to work on any other thread than this one // since namespace ops (unshare, setns) are done for a single thread, we // must ensure that the goroutine does not jump from OS thread to thread - runtime.LockOSThread() - defer runtime.UnlockOSThread() - - err := RunContainerArping(net_ns_path, pg_ifc_prefix) - if err != nil { - Log.Printf("Error while running arping : %v", err) + for i := 0; i <= 4; i++ { + runtime.LockOSThread() + err := RunContainerArping(net_ns_path, pg_ifc_prefix) + if err != nil { + Log.Printf("Arping failed : %v", err) + } + runtime.UnlockOSThread() + time.Sleep(1 * time.Second) } }(j.SandboxKey, ifname.DstPrefix) diff --git a/plugin/driver/network_utils.go b/plugin/driver/network_utils.go index c975b29..3d5ca67 100644 --- a/plugin/driver/network_utils.go +++ b/plugin/driver/network_utils.go @@ -18,7 +18,6 @@ import ( "fmt" "net" "strings" - "time" "github.com/google/gopacket" "github.com/google/gopacket/layers" @@ -32,27 +31,12 @@ import ( // that match with the prefix given to libnetwork func RunContainerArping(net_ns_path string, pg_ifc_prefix string) error { - Log.Printf("Attempting to acces the container %s NetNS", net_ns_path) var netns ns.NetNS var err error - success := false - for attempts := 0; attempts <= 5; attempts++ { - - netns, err = ns.GetNS(net_ns_path) - - if err != nil { - Log.Printf("Attempt %d: failed to open netns %s: %v", attempts, net_ns_path, err) - time.Sleep(1 * time.Second) - } else { - success = true - break - } - attempts += 1 - } - - if !success { - return fmt.Errorf("Attempts exhaused, could not access %s, exiting", net_ns_path) + netns, err = ns.GetNS(net_ns_path) + if err != nil { + return fmt.Errorf("Failed to open netns %s: %v", net_ns_path, err) } defer netns.Close() @@ -70,8 +54,6 @@ func RunContainerArping(net_ns_path string, pg_ifc_prefix string) error { if strings.HasPrefix(ifc.Name, pg_ifc_prefix) { // Valid PG-created ifc - // TODO we might want to add a retry here to make sure, - // in case the IFC is up but still has no IP, we retry address_slice, err := ifc.Addrs() if err != nil { Log.Errorf("Failed to get Address slice for ifc %s: %v", ifc.Name, err) @@ -83,7 +65,6 @@ func RunContainerArping(net_ns_path string, pg_ifc_prefix string) error { continue } - Log.Printf("Attempting raw socket open on %s", ifc.Name) handle, err := pcap.OpenLive(ifc.Name, 65536, true, pcap.BlockForever) if err != nil { return fmt.Errorf("Failed to open raw socket %s : %v", ifc.Name, err) @@ -105,18 +86,14 @@ func RunContainerArping(net_ns_path string, pg_ifc_prefix string) error { // Get ARP packet arp_pkt := getGratuitousArp(ifc.HardwareAddr, ip) - for i := 0; i <= 3; i++ { - Log.Printf("Sending %d GARP on %s : %s - %s", - i, ifc.Name, ifc.HardwareAddr.String(), ip.String()) + Log.Printf("Sending %s GARP on %s : %s", + ifc.Name, ifc.HardwareAddr.String(), ip.String()) - if err := handle.WritePacketData(arp_pkt); err != nil { - handle.Close() - return fmt.Errorf("Failed to write ARP packet data on %s : %v", ifc.Name, err) - } - time.Sleep(750 * time.Millisecond) + if err := handle.WritePacketData(arp_pkt); err != nil { + Log.Printf("Failed to write ARP packet data on %s : %v", ifc.Name, err) + continue } } - // Close the socket handle.Close() }