Ensure links don't get stuck in dialing status. Only mark link status failed if closing link was current. Fixes #1471

CHANGELOG.md

@@ -35,10 +35,21 @@ If the rate limiter is enabled, the following metrics will be produced:
 ## Component Updates and Bug Fixes
 
 * github.com/openziti/agent: [v1.0.15 -> v1.0.16](https://github.com/openziti/agent/compare/v1.0.15...v1.0.16)
-* github.com/openziti/ziti: [v0.30.5 -> v0.30.6](https://github.com/openziti/ziti/compare/v0.30.5...v0.30.6)
+* github.com/openziti/channel/v2: [v2.0.101 -> v2.0.105](https://github.com/openziti/channel/compare/v2.0.101...v2.0.105)
+* github.com/openziti/edge-api: [v0.25.38 -> v0.26.0](https://github.com/openziti/edge-api/compare/v0.25.38...v0.26.0)
+    * [Issue #49](https://github.com/openziti/edge-api/issues/49) - Add 429 responses to allow indicating that the server is too busy
+
+* github.com/openziti/identity: [v1.0.64 -> v1.0.66](https://github.com/openziti/identity/compare/v1.0.64...v1.0.66)
+* github.com/openziti/metrics: [v1.2.36 -> v1.2.37](https://github.com/openziti/metrics/compare/v1.2.36...v1.2.37)
+* github.com/openziti/sdk-golang: [v0.20.122 -> v0.20.123](https://github.com/openziti/sdk-golang/compare/v0.20.122...v0.20.123)
+* github.com/openziti/secretstream: [v0.1.12 -> v0.1.13](https://github.com/openziti/secretstream/compare/v0.1.12...v0.1.13)
+* github.com/openziti/storage: [v0.2.20 -> v0.2.23](https://github.com/openziti/storage/compare/v0.2.20...v0.2.23)
+* github.com/openziti/transport/v2: [v2.0.109 -> v2.0.113](https://github.com/openziti/transport/compare/v2.0.109...v2.0.113)
+* github.com/openziti/ziti: [v0.30.5 -> v0.31.0](https://github.com/openziti/ziti/compare/v0.30.5...v0.31.0)
+    * [Issue #1471](https://github.com/openziti/ziti/issues/1471) - Router links not resilient to controller crash
+    * [Issue #1468](https://github.com/openziti/ziti/issues/1468) - Quickstart quietly fails if password is < 5 characters long
     * [Issue #1445](https://github.com/openziti/ziti/issues/1445) - Add controller update guardrail
     * [Issue #1442](https://github.com/openziti/ziti/issues/1442) - Network watchdog not shutting down when controller shuts down
-    * [Issue #1468](https://github.com/openziti/ziti/issues/1468) - Quickstart was consuming controller initialization failures. Failures now cause quickstart to fail.
     * [Issue #1465](https://github.com/openziti/ziti/issues/1465) - Upgrade functions `getZiti` and `performMigration` were only functional on Mac OS, now they are functional for Linux and Mac OSs.
     * [Issue #1217](https://github.com/openziti/ziti/issues/1217) - Quickstart was improperly handling special characters in `ZITI_PWD`. Special characters are now supported for `ZITI_PWD` in quickstart functions.
 

go.mod

@@ -28,7 +28,7 @@ require (
 	github.com/golang-jwt/jwt/v5 v5.0.0
 	github.com/google/go-cmp v0.6.0
 	github.com/google/gopacket v1.1.19
-	github.com/google/uuid v1.3.1
+	github.com/google/uuid v1.4.0
 	github.com/gorilla/handlers v1.5.1
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.5.0
@@ -47,17 +47,17 @@ require (
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/natefinch/lumberjack v2.0.0+incompatible
 	github.com/openziti/agent v1.0.16
-	github.com/openziti/channel/v2 v2.0.101
+	github.com/openziti/channel/v2 v2.0.105
 	github.com/openziti/edge-api v0.26.0
 	github.com/openziti/foundation/v2 v2.0.33
-	github.com/openziti/identity v1.0.64
+	github.com/openziti/identity v1.0.66
 	github.com/openziti/jwks v1.0.3
-	github.com/openziti/metrics v1.2.36
+	github.com/openziti/metrics v1.2.37
 	github.com/openziti/runzmd v1.0.33
 	github.com/openziti/sdk-golang v0.20.123
-	github.com/openziti/secretstream v0.1.12
-	github.com/openziti/storage v0.2.20
-	github.com/openziti/transport/v2 v2.0.109
+	github.com/openziti/secretstream v0.1.13
+	github.com/openziti/storage v0.2.23
+	github.com/openziti/transport/v2 v2.0.113
 	github.com/openziti/x509-claims v1.0.3
 	github.com/openziti/xweb/v2 v2.1.0
 	github.com/openziti/ziti-db-explorer v1.1.3
@@ -76,7 +76,7 @@ require (
 	github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125
 	github.com/xeipuuv/gojsonschema v1.2.0
 	github.com/zitadel/oidc/v2 v2.7.0
-	go.etcd.io/bbolt v1.3.7
+	go.etcd.io/bbolt v1.3.8
 	golang.org/x/crypto v0.14.0
 	golang.org/x/net v0.17.0
 	golang.org/x/sync v0.4.0
@@ -186,5 +186,5 @@ require (
 	google.golang.org/appengine v1.6.7 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
-	nhooyr.io/websocket v1.8.9 // indirect
+	nhooyr.io/websocket v1.8.10 // indirect
 )

go.sum

@@ -372,8 +372,8 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
-github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
+github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
 github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
@@ -616,30 +616,30 @@ github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYr
 github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
 github.com/openziti/agent v1.0.16 h1:9Saji+8hFE1NpzP2XzDhsVJbCrDlhixoLHfOpFt5Z+U=
 github.com/openziti/agent v1.0.16/go.mod h1:zfm53+PVWoGFzjGGgQdKby5749G6VRYHe+eQJmoVKy4=
-github.com/openziti/channel/v2 v2.0.101 h1:UaveW/ogYKVtCQZfwRoMhsZhj/tvs1bD7SvH0aLaYNw=
-github.com/openziti/channel/v2 v2.0.101/go.mod h1:b9CBWpH6tnLqCHInDKL6AzMGqXdbEjsul3yVQUGENVU=
+github.com/openziti/channel/v2 v2.0.105 h1:WT2zFF7krZkYUfuXA+4tQxhYiiVWlldD3mKX3qJU9Ww=
+github.com/openziti/channel/v2 v2.0.105/go.mod h1:++bV6FFgGUNxaBu7iOkkCa6rSiueU34Kd6f2LSCrEtU=
 github.com/openziti/dilithium v0.3.3 h1:PLgQ6PMNLSTzCFbX/h98cmudgz/cU6TmjdSv5NAPD8k=
 github.com/openziti/dilithium v0.3.3/go.mod h1:vsCjI2AU/hon9e+dLhUFbCNGesJDj2ASgkySOcpmvjo=
 github.com/openziti/edge-api v0.26.0 h1:082hXjj8rnyMBZHYiB6jb4n7mCXtdMXpF2iCqZOv4IM=
 github.com/openziti/edge-api v0.26.0/go.mod h1:/e1pK92L471fvOAwE/hLX5sqBuuo+NwI8vmL04dUHsM=
 github.com/openziti/foundation/v2 v2.0.33 h1:8CP+fi4KsmzA4jDi54jibwFWWxKpd0rSiplzN9Z0Isw=
 github.com/openziti/foundation/v2 v2.0.33/go.mod h1:dWR0g3NOka3uKz9MgUHq6dmuRLmSvunkyeuOXEW/5qU=
-github.com/openziti/identity v1.0.64 h1:HwALRY1J/rNNcIAlr1OwCwTHU/rlMRaUi5TXAfZotjw=
-github.com/openziti/identity v1.0.64/go.mod h1:t/mW5mCpCbcRrssj4EpzfrmebI7+UKXGH2twll7IQIo=
+github.com/openziti/identity v1.0.66 h1:wCIVNCoiHmeicC9yM15FC0xe6jSic879ztN63fSD6hM=
+github.com/openziti/identity v1.0.66/go.mod h1:grReHVqBwhECrnrAZCxNw4ZpA2I1ox66tYfPkfGuKJY=
 github.com/openziti/jwks v1.0.3 h1:hf8wkb+Cg4nH/HM0KROFd7u+C3DkRVcFZJ7tDV+4icc=
 github.com/openziti/jwks v1.0.3/go.mod h1:t4xxq8vlXGsPn29kiQVnZBBDDnEoOFqtJoHibkJunQQ=
-github.com/openziti/metrics v1.2.36 h1:oW5YM9H8IqtFuxIyo0rMC3mTpl3rdSnDKcHp+ZTn+JM=
-github.com/openziti/metrics v1.2.36/go.mod h1:fjYG6sUC/n6VXe0nZbYGEBaopbRThBo/3xt7o9VatRQ=
+github.com/openziti/metrics v1.2.37 h1:5yWvMwQT6X43LDlNVcUtqAPJQXfKtbWSYoCIiOfXztg=
+github.com/openziti/metrics v1.2.37/go.mod h1:jIL9iilxby8tR98C18uZaSe6bRG15ItR8XF2hmMt8vs=
 github.com/openziti/runzmd v1.0.33 h1:tOyjRoUuVXIo1z1pNU32jALWkMmhzsSaDrhLtuOn3Ts=
 github.com/openziti/runzmd v1.0.33/go.mod h1:8c/uvZR/XWXQNllTq6LuTpfKL2DTNxfI2X2wYhgRwik=
 github.com/openziti/sdk-golang v0.20.123 h1:VD0xmA6fbiHZDtdQqTAKZeJ9prb66gyTVphjHSSoxlo=
 github.com/openziti/sdk-golang v0.20.123/go.mod h1:AbQs2gfbVsmL7/xXA2VTqAc84dFeQsyVkWBeWKNd1d4=
-github.com/openziti/secretstream v0.1.12 h1:N78CHxtqWzSyNFOsYtYRWNNTfX1ZDAPkFgzHobpodZU=
-github.com/openziti/secretstream v0.1.12/go.mod h1:gHMH1REH0r4VlmCtuWx8biU7j5ZfOivFjz9mLgwq7mk=
-github.com/openziti/storage v0.2.20 h1:xpLczyF/czIw76M4Rrt2urYn/EvGNor+SPzoixuOkLs=
-github.com/openziti/storage v0.2.20/go.mod h1:UO8D6h4AAf5OT1iJg1sppKoEPC2YWaBGZFLK7rPyk5M=
-github.com/openziti/transport/v2 v2.0.109 h1:12ZdM9R7lETKP7cT2xsGlTX/4w2qBtC8nThxw8RG77Q=
-github.com/openziti/transport/v2 v2.0.109/go.mod h1:r1jay/cSzkw15SJLtbk/I9YZtMIYjhueqAqfWtO5ioE=
+github.com/openziti/secretstream v0.1.13 h1:grp53Q5gCFPXv6okwWHDVvqBBk2BhD0ikHwfV3Adhnc=
+github.com/openziti/secretstream v0.1.13/go.mod h1:M4DYavDc3TVF/eemNqp5Fa+zGuYTNa0HTGSz/GkgUzA=
+github.com/openziti/storage v0.2.23 h1:R5ZBGDGC/LvOz3fE/GlevwbPZ3HL7VxYEvlhKuezvNU=
+github.com/openziti/storage v0.2.23/go.mod h1:NZCrN2dLtRU73McVEflK5prDgYds9J54mMNz5DmgvZE=
+github.com/openziti/transport/v2 v2.0.113 h1:xFPd1W00KqkFb62rRsRXmLqfgr9d9uk0CAVZegvtGhA=
+github.com/openziti/transport/v2 v2.0.113/go.mod h1:TSDHV7RTGg/FinzfOP8cg86O53BCabXedANh3eUNics=
 github.com/openziti/x509-claims v1.0.3 h1:HNdQ8Nf1agB3lBs1gahcO6zfkeS4S5xoQ2/PkY4HRX0=
 github.com/openziti/x509-claims v1.0.3/go.mod h1:Z0WIpBm6c4ecrpRKrou6Gk2wrLWxJO/+tuUwKh8VewE=
 github.com/openziti/xweb/v2 v2.1.0 h1:Xhh3C2pZkq/Prr65V+SfFSibLDYteoc4f62KQCcTZF4=
@@ -867,8 +867,8 @@ github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQ
 github.com/zitadel/oidc/v2 v2.7.0 h1:IGX4EDk6tegTjUSsZDWeTfLseFU0BdJ/Glf1tgys2lU=
 github.com/zitadel/oidc/v2 v2.7.0/go.mod h1:zkUkVJS0sDVy9m0UA9RgO3f8i/C0rtjvXU36UJj7T+0=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
-go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
+go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA=
+go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
 go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
 go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
 go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
@@ -1423,8 +1423,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-nhooyr.io/websocket v1.8.9 h1:+U/9DCNIH1XnzrWKs7yZp4jO0e/m6mUEh2kRPKRQYeg=
-nhooyr.io/websocket v1.8.9/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c=
+nhooyr.io/websocket v1.8.10 h1:mv4p+MnGrLDcPlBoWsvPP7XCzTYMXP9F9eIGoKbgx7Q=
+nhooyr.io/websocket v1.8.10/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/goversion v1.2.0 h1:SPn+NLTiAG7w30IRK/DKp1BjvpWabYgxlLp/+kx5J8w=
 rsc.io/goversion v1.2.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo=

router/forwarder/forwarder.go

@@ -18,14 +18,14 @@ package forwarder
 
 import (
 	"github.com/michaelquigley/pfxlog"
+	"github.com/openziti/foundation/v2/errorz"
+	"github.com/openziti/foundation/v2/info"
+	"github.com/openziti/metrics"
 	"github.com/openziti/ziti/common/inspect"
 	"github.com/openziti/ziti/common/pb/ctrl_pb"
 	"github.com/openziti/ziti/common/trace"
 	"github.com/openziti/ziti/router/xgress"
 	"github.com/openziti/ziti/router/xlink"
-	"github.com/openziti/foundation/v2/errorz"
-	"github.com/openziti/foundation/v2/info"
-	"github.com/openziti/metrics"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"time"
@@ -110,14 +110,12 @@ func (forwarder *Forwarder) HasDestination(address xgress.Address) bool {
 }
 
 func (forwarder *Forwarder) RegisterLink(link xlink.LinkDestination) error {
-	if !forwarder.destinations.addDestinationIfAbsent(xgress.Address(link.Id()), link) {
-		return errors.Errorf("unable to register link %v as it is already registered", link.Id())
-	}
+	forwarder.destinations.addDestination(xgress.Address(link.Id()), link)
 	return nil
 }
 
 func (forwarder *Forwarder) UnregisterLink(link xlink.LinkDestination) {
-	forwarder.destinations.removeDestination(xgress.Address(link.Id()))
+	forwarder.destinations.removeDestinationIfMatches(xgress.Address(link.Id()), link)
 }
 
 func (forwarder *Forwarder) Route(ctrlId string, route *ctrl_pb.Route) error {

router/forwarder/tables.go

@@ -115,10 +115,6 @@ func (dt *destinationTable) addDestination(addr xgress.Address, destination Dest
 	dt.destinations.Set(string(addr), destination)
 }
 
-func (dt *destinationTable) addDestinationIfAbsent(addr xgress.Address, destination Destination) bool {
-	return dt.destinations.SetIfAbsent(string(addr), destination)
-}
-
 func (dt *destinationTable) getDestination(addr xgress.Address) (Destination, bool) {
 	if dst, found := dt.destinations.Get(string(addr)); found {
 		return dst, true
@@ -130,6 +126,12 @@ func (dt *destinationTable) removeDestination(addr xgress.Address) {
 	dt.destinations.Remove(string(addr))
 }
 
+func (dt *destinationTable) removeDestinationIfMatches(addr xgress.Address, destination Destination) {
+	dt.destinations.RemoveCb(string(addr), func(key string, v Destination, exists bool) bool {
+		return exists && destination == v
+	})
+}
+
 func (dt *destinationTable) linkDestinationToCircuit(circuitId string, address xgress.Address) {
 	var addresses []xgress.Address
 	if i, found := dt.xgress.Get(circuitId); found {

router/handler_link/close.go

@@ -52,6 +52,8 @@ func (self *closeHandler) HandleClose(ch channel.Channel) {
 			WithField("linkId", self.link.Id()).
 			WithField("routerId", self.link.DestinationId())
 
+		self.forwarder.UnregisterLink(self.link)
+
 		// ensure that both parts of a split link are closed, if one side closes
 		go func() {
 			_ = self.link.Close()
@@ -72,7 +74,6 @@ func (self *closeHandler) HandleClose(ch channel.Channel) {
 			})
 		})
 
-		self.forwarder.UnregisterLink(self.link)
 		close(self.closeNotify)
 	}
 }

router/link/link_events.go

@@ -19,11 +19,11 @@ package link
 import (
 	"github.com/michaelquigley/pfxlog"
 	"github.com/openziti/channel/v2"
-	"github.com/openziti/ziti/controller/idgen"
+	"github.com/openziti/foundation/v2/stringz"
 	"github.com/openziti/ziti/common/inspect"
 	"github.com/openziti/ziti/common/pb/ctrl_pb"
+	"github.com/openziti/ziti/controller/idgen"
 	"github.com/openziti/ziti/router/xlink"
-	"github.com/openziti/foundation/v2/stringz"
 	"github.com/pkg/errors"
 	"sync/atomic"
 	"time"
@@ -255,7 +255,7 @@ func (self *updateLinkState) Handle(registry *linkRegistryImpl) {
 	}
 
 	state.status = self.status
-	if state.status == StatusQueueFailed || state.status == StatusDialFailed {
+	if state.status == StatusDialFailed {
 		state.dialFailed(registry)
 	}
 }

router/link/link_registry.go

@@ -22,11 +22,11 @@ import (
 	"github.com/michaelquigley/pfxlog"
 	"github.com/openziti/channel/v2"
 	"github.com/openziti/channel/v2/protobufs"
+	"github.com/openziti/foundation/v2/goroutines"
 	"github.com/openziti/ziti/common/inspect"
 	"github.com/openziti/ziti/common/pb/ctrl_pb"
 	"github.com/openziti/ziti/router/env"
 	"github.com/openziti/ziti/router/xlink"
-	"github.com/openziti/foundation/v2/goroutines"
 	"github.com/sirupsen/logrus"
 	"sync"
 	"sync/atomic"
@@ -165,9 +165,9 @@ func (self *linkRegistryImpl) LinkClosed(link xlink.Xlink) {
 	defer self.Unlock()
 	if val := self.linkMap[link.Key()]; val == link {
 		delete(self.linkMap, link.Key())
+		self.updateLinkStateClosed(link) // only update link state to closed if this was the current link
 	}
 	delete(self.linkByIdMap, link.Id())
-	self.updateLinkStateClosed(link)
 }
 
 func (self *linkRegistryImpl) Shutdown() {
@@ -325,13 +325,33 @@ func (self *linkRegistryImpl) evaluateLinkStateQueue() {
 }
 
 func (self *linkRegistryImpl) evaluateDestinations() {
-	for _, dest := range self.destinations {
-		// TODO: When do we drop destinations? Should we ask the controller after the router has been
-		//       unhealthy for a while and it doesn't have any established links? Do this on exponential backoff?
-		//      Should the controller send router removed messages?
+	for destId, dest := range self.destinations {
+		hasEstablishedLinks := false
 		for _, state := range dest.linkMap {
+			// verify that links marked as established have an open link. There's a small chance that a link established
+			// and link closed could be processed out of order if the event queue is full. This way, it will eventually
+			// get fixed.
+			if state.status == StatusEstablished {
+				link, _ := self.GetLink(state.linkKey)
+				if link == nil || link.IsClosed() {
+					// If the link is not valid, allow it to be re-dialed
+					state.retryDelay = time.Duration(0)
+					state.nextDial = time.Now()
+					state.status = StatusLinkFailed
+				} else {
+					hasEstablishedLinks = true
+				}
+			}
+
 			self.evaluateLinkState(state)
 		}
+
+		// we are notified of deleted routers. In case we're unreachable while a router is deleted,
+		// we will also stop trying to contact unhealthy routers after a period. If a destination
+		// has nothing to dial, it should also be removed
+		if len(dest.linkMap) == 0 || (!dest.healthy && !hasEstablishedLinks && time.Since(dest.unhealthyAt) > 48*time.Hour) {
+			delete(self.destinations, destId)
+		}
 	}
 }
 
@@ -343,14 +363,17 @@ func (self *linkRegistryImpl) evaluateLinkState(state *linkState) {
 	if couldDial {
 		state.status = StatusDialing
 		state.dialAttempts++
+		log.Info("queuing link to dial")
 
 		err := self.env.GetLinkDialerPool().QueueOrError(func() {
 			link, _ := self.GetLink(state.linkKey)
 			if link != nil {
-				log.Warn("link already present, but link status still pending")
+				log.Info("link already present, attempting to mark established")
+				self.updateLinkStateEstablished(link)
 				return
 			}
 
+			log.Info("dialing link")
 			link, err := state.dialer.Dial(state)
 			if err != nil {
 				log.WithError(err).Error("error dialing link")
@@ -360,14 +383,23 @@ func (self *linkRegistryImpl) evaluateLinkState(state *linkState) {
 				})
 				return
 			}
-			self.DialSucceeded(link)
+
+			existing, success := self.DialSucceeded(link)
+			if !success {
+				if existing != nil {
+					self.updateLinkStateEstablished(link)
+				} else {
+					self.queueEvent(&updateLinkState{
+						linkState: state,
+						status:    StatusDialFailed,
+					})
+				}
+			}
 		})
 		if err != nil {
 			log.WithError(err).Error("unable to queue link dial, see pool error")
-			self.queueEvent(&updateLinkState{
-				linkState: state,
-				status:    StatusQueueFailed,
-			})
+			state.status = StatusQueueFailed
+			state.dialFailed(self)
 		}
 	}
 }

router/xgress_edge/hosted.go

@@ -66,7 +66,7 @@ func (registry *hostedServiceRegistry) cleanupDuplicates(newest *edgeTerminator)
 	registry.services.Range(func(key, value interface{}) bool {
 		terminator := value.(*edgeTerminator)
 		if terminator != newest && newest.token == terminator.token && newest.instance == terminator.instance {
-			terminator.close(true, "duplicate terminator") // don't notify, channel is already closed, we can't send messages
+			terminator.close(false, "duplicate terminator") // don't notify, channel is already closed, we can't send messages
 			registry.services.Delete(key)
 			pfxlog.Logger().WithField("routerId", terminator.edgeClientConn.listener.id.Token).
 				WithField("sessionToken", terminator.token).