From 58caf38f0739a96ae3bc0b2d1bcbf4b2604e08a7 Mon Sep 17 00:00:00 2001 From: Martynas Pumputis Date: Thu, 16 Jun 2016 18:40:38 +0100 Subject: [PATCH 1/2] Fix re-attaching of weave router in awsvpc mode In order to re-attach the router and make it work in awsvpc mode, the following has to be done: - expose_ip on the router - adjusting arp settings - restoring host routes To detect that we run in awsvpc mode, we inspect cmd parameters of the weave router instead of GET'ing /ipinfo/tracker. This is so, because the HTTP server started by the router might not be running at the time of restoration. --- ipam/ring/ring.go | 2 ++ ipam/tracker/awsvpc.go | 17 ++++++++--------- weave | 7 +++++++ 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/ipam/ring/ring.go b/ipam/ring/ring.go index 67d1188cf4..1d3c10b2f6 100644 --- a/ipam/ring/ring.go +++ b/ipam/ring/ring.go @@ -119,6 +119,8 @@ func New(start, end address.Address, peer mesh.PeerName, f OnUpdate) *Ring { } func (r *Ring) Restore(other *Ring) { + defer r.trackUpdates()() + onUpdate := r.onUpdate *r = *other r.onUpdate = onUpdate diff --git a/ipam/tracker/awsvpc.go b/ipam/tracker/awsvpc.go index 1089b61543..51424379ef 100644 --- a/ipam/tracker/awsvpc.go +++ b/ipam/tracker/awsvpc.go @@ -21,6 +21,7 @@ package tracker import ( "fmt" "net" + "syscall" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/ec2metadata" @@ -94,14 +95,14 @@ func (t *AWSVPCTracker) HandleUpdate(prevRanges, currRanges []address.Range, loc for _, cidr := range curr { cidrStr := cidr.String() t.debugf("adding route %s to %s", cidrStr, t.instanceID) - _, err := t.createVPCRoute(cidrStr) - if err != nil { + if _, err := t.createVPCRoute(cidrStr); err != nil { return fmt.Errorf("createVPCRoutes failed: %s", err) } if local { - err = t.createHostRoute(cidrStr) - if err != nil { - return fmt.Errorf("createHostRoute failed: %s", err) + if err := t.createHostRoute(cidrStr); err != nil { + if errno, ok := err.(syscall.Errno); !(ok && errno == syscall.EEXIST) { + return fmt.Errorf("createHostRoute failed: %s", err) + } } } } @@ -110,13 +111,11 @@ func (t *AWSVPCTracker) HandleUpdate(prevRanges, currRanges []address.Range, loc for _, cidr := range prev { cidrStr := cidr.String() t.debugf("removing %s route", cidrStr) - _, err := t.deleteVPCRoute(cidrStr) - if err != nil { + if _, err := t.deleteVPCRoute(cidrStr); err != nil { return fmt.Errorf("deleteVPCRoute failed: %s", err) } if local { - err = t.deleteHostRoute(cidrStr) - if err != nil { + if err := t.deleteHostRoute(cidrStr); err != nil { return fmt.Errorf("deleteHostRoute failed: %s", err) } } diff --git a/weave b/weave index 8c712897c6..c232a8a553 100755 --- a/weave +++ b/weave @@ -1656,6 +1656,10 @@ launch_router() { setup_router_iface_$BRIDGE_TYPE wait_for_status $CONTAINER_NAME http_call $HTTP_ADDR populate_router + setup_awsvpc +} + +setup_awsvpc() { if [ -n "$AWSVPC" ]; then expose_ip # Set proxy_arp on the bridge, so that it could accept packets destined @@ -1956,11 +1960,14 @@ EOF attach-router) check_running $CONTAINER_NAME enforce_docker_bridge_addr_assign_type + # We cannot use detect_awsvpc here, because HTTP server might not be started + ! docker inspect -f '{{.Config.Cmd}}' $CONTAINER_NAME | grep -q -- "--awsvpc" || AWSVPC=1 create_bridge fetch_router_args setup_router_iface_$BRIDGE_TYPE wait_for_status $CONTAINER_NAME http_call $HTTP_ADDR populate_router + setup_awsvpc ;; launch-proxy) deprecation_warnings "$@" From 6319860230613f41b3b52e8c8d2ce1ccef293905 Mon Sep 17 00:00:00 2001 From: Martynas Pumputis Date: Fri, 17 Jun 2016 14:45:27 +0100 Subject: [PATCH 2/2] Change the order of awsvpc initialization We want to setup the necessary arp settings as soon as possible and before we re-insert DNS entries. Otherwise, the announced containers might not be reachable. Also, do not try to reclaim for weave:expose inside populate_router, because we do expose before calling it. --- weave | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/weave b/weave index c232a8a553..ec4eaed6ab 100755 --- a/weave +++ b/weave @@ -1655,13 +1655,12 @@ launch_router() { "$@") setup_router_iface_$BRIDGE_TYPE wait_for_status $CONTAINER_NAME http_call $HTTP_ADDR - populate_router setup_awsvpc + populate_router } setup_awsvpc() { if [ -n "$AWSVPC" ]; then - expose_ip # Set proxy_arp on the bridge, so that it could accept packets destined # to containers within the same subnet but running on remote hosts. # Without it, exact routes on each container are required. @@ -1670,6 +1669,7 @@ setup_awsvpc() { # placing the request into a bounded queue as it can be seen: # https://git.kernel.org/cgit/linux/kernel/git/stable/linux-stable.git/tree/net/ipv4/arp.c?id=refs/tags/v4.6.1#n819 echo 0 >/proc/sys/net/ipv4/neigh/$BRIDGE/proxy_delay + expose_ip fi } @@ -1683,7 +1683,8 @@ fetch_router_args() { populate_router() { if [ -n "$IPRANGE" ] ; then # Tell the newly-started weave IP allocator about existing weave IPs - with_container_addresses ipam_reclaim_no_check_alive weave:expose + # In the case of AWSVPC, we do expose before calling populate_router + [ -n "$AWSVPC" ] || with_container_addresses ipam_reclaim_no_check_alive weave:expose with_container_addresses ipam_reclaim $(docker ps -q --no-trunc) fi if [ -z "$NO_DNS_OPT" ] ; then @@ -1966,8 +1967,8 @@ EOF fetch_router_args setup_router_iface_$BRIDGE_TYPE wait_for_status $CONTAINER_NAME http_call $HTTP_ADDR - populate_router setup_awsvpc + populate_router ;; launch-proxy) deprecation_warnings "$@"