Merge pull request #2305 from /issues/2186-peer-list-persistence

Peer list persistence; Fixes #2186.

prog/weaver/main.go

@@ -151,6 +151,7 @@ func main() {
 		config             mesh.Config
 		networkConfig      weave.NetworkConfig
 		protocolMinVersion int
+		resume             bool
 		ifaceName          string
 		routerName         string
 		nickName           string
@@ -181,6 +182,7 @@ func main() {
 	mflag.StringVar(&config.Host, []string{"-host"}, "", "router host")
 	mflag.IntVar(&config.Port, []string{"#port", "-port"}, mesh.Port, "router port")
 	mflag.IntVar(&protocolMinVersion, []string{"-min-protocol-version"}, mesh.ProtocolMinVersion, "minimum weave protocol version")
+	mflag.BoolVar(&resume, []string{"-resume"}, false, "resume connections to previous peers")
 	mflag.StringVar(&ifaceName, []string{"#iface", "-iface"}, "", "name of interface to capture/inject from (disabled if blank)")
 	mflag.StringVar(&routerName, []string{"#name", "-name"}, "", "name of router (defaults to MAC of interface)")
 	mflag.StringVar(&nickName, []string{"#nickname", "-nickname"}, "", "nickname of peer (defaults to hostname)")
@@ -218,6 +220,9 @@ func main() {
 	mflag.Parse()
 
 	peers = mflag.Args()
+	if resume && len(peers) > 0 {
+		Log.Fatalf("You must not specify an initial peer list in conjuction with --resume")
+	}
 
 	common.SetLogLevel(logLevel)
 
@@ -267,7 +272,7 @@ func main() {
 	router := weave.NewNetworkRouter(config, networkConfig, name, nickName, overlay, db)
 	Log.Println("Our name is", router.Ourself)
 
-	if peers, err = router.InitialPeers(peers); err != nil {
+	if peers, err = router.InitialPeers(resume, peers); err != nil {
 		Log.Fatal("Unable to get initial peer set: ", err)
 	}
 

router/network_router.go

@@ -1,8 +1,10 @@
 package router
 
 import (
+	"fmt"
 	"math"
 	"net"
+	"os"
 	"time"
 
 	"github.com/weaveworks/mesh"
@@ -211,11 +213,10 @@ func (router *NetworkRouter) relayBroadcast(srcPeer *mesh.Peer, key PacketKey) F
 const peersIdent = "directPeers"
 
 func (router *NetworkRouter) persistPeers() {
-	// Persistence disabled pending rationalisation of the expected behaviour
-	/*if err := router.db.Save(peersIdent, router.ConnectionMaker.Targets(false)); err != nil {
+	if err := router.db.Save(peersIdent, router.ConnectionMaker.Targets(false)); err != nil {
 		log.Errorf("Error persisting peers: %s", err)
 		return
-	}*/
+	}
 }
 
 func (router *NetworkRouter) InitiateConnections(peers []string, replace bool) []error {
@@ -229,30 +230,22 @@ func (router *NetworkRouter) ForgetConnections(peers []string) {
 	router.persistPeers()
 }
 
-func (router *NetworkRouter) InitialPeers(peers []string) ([]string, error) {
-	var storedPeers []string
-	// Persistence disabled pending rationalisation of the expected behaviour
-	/*if _, err := router.db.Load(peersIdent, &storedPeers); err != nil {
-		return nil, err
-	}*/
-
-	if storedPeers != nil && !equal(peers, storedPeers) {
-		log.Println("Overriding initial peer list with stored list:", storedPeers)
-		peers = storedPeers
-	} else {
-		log.Println("Initial set of peers:", peers)
+func (router *NetworkRouter) InitialPeers(resume bool, peers []string) ([]string, error) {
+	if _, err := os.Stat("restart.sentinel"); err == nil || resume {
+		var storedPeers []string
+		if _, err := router.db.Load(peersIdent, &storedPeers); err != nil {
+			return nil, err
+		}
+		log.Println("Restart/resume detected - using persisted peer list:", storedPeers)
+		return storedPeers, nil
 	}
-	return peers, nil
-}
 
-func equal(a, b []string) bool {
-	if len(a) != len(b) {
-		return false
+	sentinel, err := os.Create("restart.sentinel")
+	if err != nil {
+		return nil, fmt.Errorf("error creating sentinel: %v", err)
 	}
-	for i, v := range a {
-		if v != b[i] {
-			return false
-		}
-	}
-	return true
+	sentinel.Close()
+
+	log.Println("Launch detected - using supplied peer list:", peers)
+	return peers, nil
 }

site/using-weave/finding-adding-hosts-dynamically.md

@@ -3,58 +3,69 @@ title: Adding and Removing Hosts Dynamically
 menu_order: 90
 ---
 
+To add a host to an existing Weave network, launch Weave Net on the
+host, and supply the address of at least one host. Weave Net
+automatically discovers any other hosts in the network and establishes
+connections with them if it can (in order to avoid unnecessary
+multi-hop routing).
 
-To add a host to an existing Weave network, simply launch 
-Weave Net on the host, and then supply the address of at least 
-one host. Weave Net automatically discovers any other hosts in 
-the network and establishes connections with them if it 
-can (in order to avoid unnecessary multi-hop routing).
+In some situations existing Weave Net hosts may be unreachable from
+the new host due to firewalls, etc.  However, it is still possible to
+add the new host, provided that inverse connections, for example, from
+existing hosts to the new hosts, are available.
 
-In some situations all existing Weave Net hosts may be 
-unreachable from the new host due to firewalls, etc. 
-However, it is still possible to add the new host, 
-provided that inverse connections, for example, 
-from existing hosts to the new hosts, are available. 
-
-To accomplish this, launch Weave Net onto the new host 
-without supplying any additional addresses.  And then, from one 
-of the existing hosts run:
+To accomplish this, launch Weave Net onto the new host without
+supplying any additional addresses and then, from one of the existing
+hosts run:
 
     host# weave connect $NEW_HOST
 
-Other hosts on the Weave network will automatically attempt
-to establish connections to the new host as well. 
+Any other existing hosts on the Weave network will attempt to
+establish connections to the new host as well.
+
+###Instructing Peers to Forget a Host
 
-Alternatively, you can also instruct a peer to forget a 
-particular host specified to it via `weave launch` or 
-`weave connect` by running:
+To instruct a peer to forget a particular host specified to it via
+`weave launch` or `weave connect` run:
 
     host# weave forget $DECOMMISSIONED_HOST
 
-This prevents the peer from trying to reconnect to that host 
-once connectivity to it is lost, and therefore can be used 
-to administratively remove any decommissioned peers 
-from the network.
+This prevents the peer from reconnecting to that host once
+connectivity to it is lost, and can be used to administratively remove
+any decommissioned peers from the network.
 
-Hosts may also be bulk-replaced. All existing hosts 
-will be forgotten, and the new hosts will be added:
+###Bulk Replacing Hosts
+
+Hosts can also be bulk-replaced. All existing hosts will be forgotten,
+and the new hosts added:
 
     host# weave connect --replace $NEW_HOST1 $NEW_HOST2
 
-For complete control over the peer topology, automatic 
-discovery can be disabled using the `--no-discovery` 
-option with `weave launch`. 
+###Restarting Docker and Weave Net
+
+If Weave Net is restarted by Docker it automatically remembers any
+previous connect and forget operations, however if you stop it
+manually and launch it again, it will not remember any prior connects.
+If you want to launch again and retain the results of those operations
+use `--resume`:
+
+    host# weave launch --resume
+
+> **Note:** In this case, you cannot specify a list of addresses,
+> since the previous peer list is used exclusively.
+
+For complete control over the peer topology, disable automatic
+discovery using the `--no-discovery` option with `weave launch`.
 
-If discovery if disabled, Weave Net only connects to the 
-addresses specified at launch time and with `weave connect`.
+If discovery if disabled, Weave Net only connects to the addresses
+specified at launch time and with `weave connect`.
 
-A list of all hosts that a peer has been asked to connect 
-to with `weave launch` and `weave connect` 
-can be obtained using:
+To return a list of all hosts and their peer connections established
+with `weave launch` and `weave connect` run:
 
     host# weave status targets
 
-**See Also** 
+**See Also**
 
  * [Enabling Multi-Cloud, Multi-Hop Networking and Routing](/site/using-weave/multi-cloud-multi-hop.md)
  * [Stopping and Removing Peers](/site/ipam/stop-remove-peers-ipam.md)

test/090_docker_restart_policy_2_test.sh

@@ -4,24 +4,6 @@
 
 start_suite "Test docker restart policy"
 
-check_restart() {
-    OLD_PID=$(container_pid $1 $2)
-
-    run_on $1 sudo kill $OLD_PID
-
-    for i in $(seq 1 10); do
-        NEW_PID=$(container_pid $1 $2)
-
-        if [ $NEW_PID != 0 -a $NEW_PID != $OLD_PID ] ; then
-            return 0
-        fi
-
-        sleep 1
-    done
-
-    return 1
-}
-
 weave_on $HOST1 launch
 
 assert "docker_on $HOST1 inspect -f '{{.HostConfig.RestartPolicy.Name}}' weave weaveproxy weaveplugin" "always\nalways\nalways"

test/175_target_peer_persistence_3_test.sh

@@ -0,0 +1,36 @@
+#! /bin/bash
+
+. ./config.sh
+
+
+assert_targets() {
+    HOST=$1
+    shift
+    EXPECTED=$(for TARGET in $@; do echo $TARGET; done | sort)
+    assert "weave_on $HOST report | jq -r '.Router.Targets[] | tostring' | sort" "$EXPECTED"
+}
+
+start_suite "Check Docker restart uses persisted peer list"
+
+# Launch router and modify initial peer list
+weave_on $HOST1 launch $HOST1 $HOST2
+weave_on $HOST1 forget $HOST1
+weave_on $HOST1 connect $HOST3
+
+# Ensure modified peer list is still in effect after restart
+check_restart $HOST1 weave
+assert_targets $HOST1 $HOST2 $HOST3
+
+# Ensure persisted peer changes are still in effect after --resume
+weave_on $HOST1 stop
+weave_on $HOST1 launch --resume
+assert_targets $HOST1 $HOST2 $HOST3
+
+# Ensure persisted peer changes are ignored after stop and subsequent restart
+weave_on $HOST1 stop
+weave_on $HOST1 launch $HOST1 $HOST2
+assert_targets $HOST1 $HOST1 $HOST2
+check_restart $HOST1 weave
+assert_targets $HOST1 $HOST1 $HOST2
+
+end_suite

test/config.sh

@@ -249,6 +249,25 @@ assert_dns_ptr_record() {
     assert "exec_on $1 $2 getent hosts $4 | tr -s ' '" "$4 $3"
 }
 
+# Kill a container process and make sure it's restarted by Docker
+check_restart() {
+    OLD_PID=$(container_pid $1 $2)
+
+    run_on $1 sudo kill $OLD_PID
+
+    for i in $(seq 1 10); do
+        NEW_PID=$(container_pid $1 $2)
+
+        if [ $NEW_PID != 0 -a $NEW_PID != $OLD_PID ] ; then
+            return 0
+        fi
+
+        sleep 1
+    done
+
+    return 1
+}
+
 start_suite() {
     for host in $HOSTS; do
         [ -z "$DEBUG" ] || echo "Cleaning up on $host: removing all containers and resetting weave"