proxy: reliability fixes

Author: xcolwell

proxy/main.go

@@ -45,6 +45,7 @@ func DefaultProxySettings() *ProxySettings {
 		ProxyWriteTimeout:        30 * time.Second,
 		ProxyIdleTimeout:         5 * time.Minute,
 		ProxyTlsHandshakeTimeout: 30 * time.Second,
+		ProxyConnectTimeout:      30 * time.Minute,
 		NotificationTimeout:      5 * time.Second,
 		WarmupTimeout:            30 * time.Minute,
 	}
@@ -55,6 +56,7 @@ type ProxySettings struct {
 	ProxyWriteTimeout        time.Duration
 	ProxyIdleTimeout         time.Duration
 	ProxyTlsHandshakeTimeout time.Duration
+	ProxyConnectTimeout      time.Duration
 	NotificationTimeout      time.Duration
 	WarmupTimeout            time.Duration
 }
@@ -411,6 +413,7 @@ func (self *httpServer) run() {
 	httpProxy.ProxyReadTimeout = self.settings.ProxyReadTimeout
 	httpProxy.ProxyWriteTimeout = self.settings.ProxyWriteTimeout
 	httpProxy.ProxyIdleTimeout = self.settings.ProxyIdleTimeout
+	httpProxy.ProxyConnectTimeout = self.settings.ProxyConnectTimeout
 	httpProxy.ProxyTlsHandshakeTimeout = self.settings.ProxyTlsHandshakeTimeout
 	// httpProxy.AllowHTTP2 = true
 	// httpProxy.Tr = &http.Transport{

proxy/proxy_device.go

@@ -35,7 +35,7 @@ type ProxyDeviceManager struct {
 	networkSpace *sdk.NetworkSpace
 
 	stateLock    sync.Mutex
-	proxyDevices map[server.Id]*ProxyDevice
+	proxyDevices map[server.Id]*proxyDeviceState
 }
 
 func NewProxyDeviceManagerWithDefaults(ctx context.Context) *ProxyDeviceManager {
@@ -62,14 +62,11 @@ func NewProxyDeviceManager(ctx context.Context, settings *ProxyDeviceManagerSett
 		cancel:       cancel,
 		settings:     settings,
 		networkSpace: networkSpace,
-		proxyDevices: map[server.Id]*ProxyDevice{},
+		proxyDevices: map[server.Id]*proxyDeviceState{},
 	}
 }
 
 func (self *ProxyDeviceManager) OpenProxyDevice(proxyId server.Id) (*ProxyDevice, error) {
-	self.stateLock.Lock()
-	defer self.stateLock.Unlock()
-
 	nextProxyDevice := func() (*ProxyDevice, error) {
 		proxyDeviceConfig := model.GetProxyDeviceConfig(self.ctx, proxyId)
 		if proxyDeviceConfig == nil {
@@ -85,18 +82,25 @@ func (self *ProxyDeviceManager) OpenProxyDevice(proxyId server.Id) (*ProxyDevice
 			defer func() {
 				self.stateLock.Lock()
 				defer self.stateLock.Unlock()
+
 				glog.Infof("[pd]cancel")
 				// note we don't call close here because only the sender should call close
 				pd.Cancel()
-				if currentPd := self.proxyDevices[proxyId]; pd == currentPd {
-					delete(self.proxyDevices, proxyId)
+				if pdState, ok := self.proxyDevices[proxyId]; ok {
+					func() {
+						pdState.StateLock.Lock()
+						defer pdState.StateLock.Unlock()
+
+						if pd == pdState.ProxyDevice {
+							delete(self.proxyDevices, proxyId)
+						}
+					}()
 				}
 			}()
 			pd.Run()
 		})
 
 		go server.HandleError(func() {
-			defer self.cancel()
 			for {
 				if pd.CancelIfIdle() {
 					return
@@ -113,20 +117,35 @@ func (self *ProxyDeviceManager) OpenProxyDevice(proxyId server.Id) (*ProxyDevice
 		return pd, nil
 	}
 
-	if pd, ok := self.proxyDevices[proxyId]; ok && pd.UpdateActivity() {
-		return pd, nil
+	pdState := func() *proxyDeviceState {
+		self.stateLock.Lock()
+		defer self.stateLock.Unlock()
+		pdState, ok := self.proxyDevices[proxyId]
+		if !ok {
+			pdState = &proxyDeviceState{}
+			self.proxyDevices[proxyId] = pdState
+		}
+		return pdState
+	}()
+
+	pdState.StateLock.Lock()
+	defer pdState.StateLock.Unlock()
+
+	if pd := pdState.ProxyDevice; pd != nil {
+		if pd.UpdateActivity() {
+			return pd, nil
+		} else {
+			pd.Cancel()
+			pdState.ProxyDevice = nil
+		}
 	}
 
 	pd, err := nextProxyDevice()
 	if err != nil {
 		return nil, err
 	}
 
-	if replacedPd, ok := self.proxyDevices[proxyId]; ok {
-		replacedPd.Cancel()
-	}
-	self.proxyDevices[proxyId] = pd
-
+	pdState.ProxyDevice = pd
 	return pd, nil
 }
 
@@ -139,6 +158,11 @@ func (self *ProxyDeviceManager) Close() {
 	self.cancel()
 }
 
+type proxyDeviceState struct {
+	StateLock   sync.Mutex
+	ProxyDevice *ProxyDevice
+}
+
 func DefaultProxyDeviceSettings() *ProxyDeviceSettings {
 	return &ProxyDeviceSettings{
 		ProxyDeviceDescription: "resident proxy",