add retry for by "no such host" failed request

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.15-alpine AS builder
+FROM golang:1.16-alpine AS builder
 
 ARG BUILD_VERSION
 

go.mod

@@ -1,6 +1,6 @@
 module github.com/alcounit/selenosis
 
-go 1.15
+go 1.16
 
 require (
 	github.com/fsnotify/fsnotify v1.4.9

go.sum

@@ -86,7 +86,6 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -261,7 +260,6 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -366,7 +364,6 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -458,7 +455,6 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

handlers.go

@@ -58,7 +58,7 @@ type response struct {
 	Selenosis Status `json:"selenosis,omitempty"`
 }
 
-//HandleSession ...
+// HandleSession ...
 func (app *App) HandleSession(w http.ResponseWriter, r *http.Request) {
 	start := time.Now()
 	logger := app.logger.WithFields(logrus.Fields{
@@ -116,15 +116,24 @@ func (app *App) HandleSession(w http.ResponseWriter, r *http.Request) {
 	logger.WithField("time_elapsed", tools.TimeElapsed(start)).Infof("starting browser from image: %s", browser.Image)
 
 	image := parseImage(browser.Image)
-	service, err := app.client.Service().Create(platform.ServiceSpec{
-		SessionID:             fmt.Sprintf("%s-%s", image, uuid.New()),
-		RequestedCapabilities: caps,
-		Template:              browser,
-	})
-	if err != nil {
-		logger.WithField("time_elapsed", tools.TimeElapsed(start)).Errorf("failed to start browser: %v", err)
-		tools.JSONError(w, err.Error(), http.StatusBadRequest)
-		return
+
+	var service platform.Service
+	j := 1
+	for ; ; j++ {
+		service, err = app.client.Service().Create(platform.ServiceSpec{
+			SessionID:             fmt.Sprintf("%s-%s", image, uuid.New()),
+			RequestedCapabilities: caps,
+			Template:              browser,
+		})
+		if err != nil {
+			logger.WithField("time_elapsed", tools.TimeElapsed(start)).Errorf("failed to start browser: %v", err)
+			if j < app.sessionRetryCount {
+				continue
+			}
+			tools.JSONError(w, "failed to start browser: "+err.Error(), http.StatusBadRequest)
+			return
+		}
+		break
 	}
 
 	cancel := func() {
@@ -152,6 +161,7 @@ func (app *App) HandleSession(w http.ResponseWriter, r *http.Request) {
 			case context.DeadlineExceeded:
 				logger.WithField("time_elapsed", tools.TimeElapsed(start)).Warn("session attempt timeout")
 				if i < app.sessionRetryCount {
+					logger.WithField("time_elapsed", tools.TimeElapsed(start)).Warnf("session retrying: %d/%d", i, app.sessionRetryCount)
 					continue
 				}
 				logger.WithField("time_elapsed", tools.TimeElapsed(start)).Warn("service is not ready")
@@ -165,6 +175,10 @@ func (app *App) HandleSession(w http.ResponseWriter, r *http.Request) {
 		}
 		if err != nil {
 			logger.WithField("time_elapsed", tools.TimeElapsed(start)).Errorf("session failed: %v", err)
+			if i < app.sessionRetryCount {
+				logger.WithField("time_elapsed", tools.TimeElapsed(start)).Warnf("session retrying for session failed: %d/%d", i, app.sessionRetryCount)
+				continue
+			}
 			tools.JSONError(w, "New session attempts retry count exceeded", http.StatusInternalServerError)
 			cancel()
 			return
@@ -194,7 +208,7 @@ func (app *App) HandleSession(w http.ResponseWriter, r *http.Request) {
 
 }
 
-//HandleProxy ...
+// HandleProxy ...
 func (app *App) HandleProxy(w http.ResponseWriter, r *http.Request) {
 	sessionID, ok := mux.Vars(r)["sessionId"]
 	if !ok {
@@ -215,23 +229,54 @@ func (app *App) HandleProxy(w http.ResponseWriter, r *http.Request) {
 		"request":    fmt.Sprintf("%s %s", r.Method, r.URL.Path),
 	})
 
-	(&httputil.ReverseProxy{
-		Director: func(r *http.Request) {
-			r.URL.Scheme = "http"
-			r.Host = sessionID + "." + app.serviceName + ":" + app.sidecarPort
-			r.URL.Host = r.Host
-			r.Header.Set("X-Forwarded-Selenosis", app.selenosisHost)
-			logger.Info("proxying session")
-		},
-		ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) {
-			logger.Errorf("proxying session error: %v", err)
-			w.WriteHeader(http.StatusBadGateway)
-		},
-	}).ServeHTTP(w, r)
+	r.URL.Scheme = "http"
+	r.Host = sessionID + "." + app.serviceName + ":" + app.sidecarPort
+	r.URL.Host = r.Host
+	r.Header.Set("X-Forwarded-Selenosis", app.selenosisHost)
+
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		logger.Errorf("Body readform err: %v", err)
+	}
+
+	i := 1
+	for ; ; i++ {
+
+		rCopy := r.Clone(r.Context())
 
+		r.Body = ioutil.NopCloser(bytes.NewReader(body))
+		rCopy.Body = ioutil.NopCloser(bytes.NewReader(body))
+		r.ParseForm()
+
+		retryLoop := true
+		revp := (&httputil.ReverseProxy{
+			Director: func(rCopy *http.Request) {
+				logger.Infof("proxying session -> Body=%v", string(body))
+				retryLoop = true
+			},
+			ErrorHandler: func(w http.ResponseWriter, rCopy *http.Request, err error) {
+				retryLoop = false
+				logger.Errorf("proxying session error (%d/%d): %v", i, app.sessionRetryCount, err)
+				if !strings.Contains(err.Error(), "no such host") || i == app.sessionRetryCount {
+					retryLoop = true
+					if strings.Contains(err.Error(), "no such host") {
+						tools.JSONError(w, err.Error(), http.StatusBadRequest)
+					} else {
+						tools.JSONError(w, fmt.Sprintf("proxing session error: %v", err.Error()), http.StatusBadRequest)
+					}
+				}
+			},
+		})
+
+		revp.ServeHTTP(w, rCopy)
+
+		if retryLoop || i > app.sessionRetryCount {
+			break
+		}
+	}
 }
 
-//HandleHubStatus ...
+// HandleHubStatus ...
 func (app *App) HandleHubStatus(w http.ResponseWriter, _ *http.Request) {
 	w.Header().Set("Content-Type", "application/json")
 
@@ -244,7 +289,7 @@ func (app *App) HandleHubStatus(w http.ResponseWriter, _ *http.Request) {
 		})
 }
 
-//HandleReverseProxy ...
+// HandleReverseProxy ...
 func (app *App) HandleReverseProxy(w http.ResponseWriter, r *http.Request) {
 	sessionID, ok := mux.Vars(r)["sessionId"]
 	if !ok {
@@ -281,7 +326,7 @@ func (app *App) HandleReverseProxy(w http.ResponseWriter, r *http.Request) {
 	}).ServeHTTP(w, r)
 }
 
-//HandleVNC ...
+// HandleVNC ...
 func (app *App) HandleVNC() websocket.Handler {
 	return func(wsconn *websocket.Conn) {
 		defer wsconn.Close()
@@ -322,7 +367,7 @@ func (app *App) HandleVNC() websocket.Handler {
 	}
 }
 
-//HandleLogs ...
+// HandleLogs ...
 func (app *App) HandleLogs() websocket.Handler {
 	return func(wsconn *websocket.Conn) {
 		defer wsconn.Close()
@@ -363,7 +408,7 @@ func (app *App) HandleLogs() websocket.Handler {
 	}
 }
 
-//HandleStatus ...
+// HandleStatus ...
 func (app *App) HandleStatus(w http.ResponseWriter, _ *http.Request) {
 	w.Header().Set("Content-Type", "application/json")