From 8e248f29ce8fe3a91b047bd2800873793ab97583 Mon Sep 17 00:00:00 2001 From: Abdullah Sarikaya Date: Wed, 28 Feb 2024 01:05:25 +0300 Subject: [PATCH] add retry for failed request --- Dockerfile | 2 +- go.mod | 2 +- go.sum | 4 -- handlers.go | 103 +++++++++++++++++++++++++++++++++++++--------------- 4 files changed, 76 insertions(+), 35 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5e3257f..e716b14 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.15-alpine AS builder +FROM golang:1.16-alpine AS builder ARG BUILD_VERSION diff --git a/go.mod b/go.mod index 53ddfda..04b4bd6 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/alcounit/selenosis -go 1.15 +go 1.16 require ( github.com/fsnotify/fsnotify v1.4.9 diff --git a/go.sum b/go.sum index dd3ad9d..1de30ff 100644 --- a/go.sum +++ b/go.sum @@ -86,7 +86,6 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -261,7 +260,6 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -366,7 +364,6 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -458,7 +455,6 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/handlers.go b/handlers.go index 953d4f2..71c5eb2 100644 --- a/handlers.go +++ b/handlers.go @@ -58,7 +58,7 @@ type response struct { Selenosis Status `json:"selenosis,omitempty"` } -//HandleSession ... +// HandleSession ... func (app *App) HandleSession(w http.ResponseWriter, r *http.Request) { start := time.Now() logger := app.logger.WithFields(logrus.Fields{ @@ -116,15 +116,24 @@ func (app *App) HandleSession(w http.ResponseWriter, r *http.Request) { logger.WithField("time_elapsed", tools.TimeElapsed(start)).Infof("starting browser from image: %s", browser.Image) image := parseImage(browser.Image) - service, err := app.client.Service().Create(platform.ServiceSpec{ - SessionID: fmt.Sprintf("%s-%s", image, uuid.New()), - RequestedCapabilities: caps, - Template: browser, - }) - if err != nil { - logger.WithField("time_elapsed", tools.TimeElapsed(start)).Errorf("failed to start browser: %v", err) - tools.JSONError(w, err.Error(), http.StatusBadRequest) - return + + var service platform.Service + j := 1 + for ; ; j++ { + service, err = app.client.Service().Create(platform.ServiceSpec{ + SessionID: fmt.Sprintf("%s-%s", image, uuid.New()), + RequestedCapabilities: caps, + Template: browser, + }) + if err != nil { + logger.WithField("time_elapsed", tools.TimeElapsed(start)).Errorf("failed to start browser: %v", err) + if j < app.sessionRetryCount { + continue + } + tools.JSONError(w, "failed to start browser: "+err.Error(), http.StatusBadRequest) + return + } + break } cancel := func() { @@ -152,6 +161,7 @@ func (app *App) HandleSession(w http.ResponseWriter, r *http.Request) { case context.DeadlineExceeded: logger.WithField("time_elapsed", tools.TimeElapsed(start)).Warn("session attempt timeout") if i < app.sessionRetryCount { + logger.WithField("time_elapsed", tools.TimeElapsed(start)).Warnf("session retrying: %d/%d", i, app.sessionRetryCount) continue } logger.WithField("time_elapsed", tools.TimeElapsed(start)).Warn("service is not ready") @@ -165,6 +175,10 @@ func (app *App) HandleSession(w http.ResponseWriter, r *http.Request) { } if err != nil { logger.WithField("time_elapsed", tools.TimeElapsed(start)).Errorf("session failed: %v", err) + if i < app.sessionRetryCount { + logger.WithField("time_elapsed", tools.TimeElapsed(start)).Warnf("session retrying for session failed: %d/%d", i, app.sessionRetryCount) + continue + } tools.JSONError(w, "New session attempts retry count exceeded", http.StatusInternalServerError) cancel() return @@ -194,7 +208,7 @@ func (app *App) HandleSession(w http.ResponseWriter, r *http.Request) { } -//HandleProxy ... +// HandleProxy ... func (app *App) HandleProxy(w http.ResponseWriter, r *http.Request) { sessionID, ok := mux.Vars(r)["sessionId"] if !ok { @@ -215,23 +229,54 @@ func (app *App) HandleProxy(w http.ResponseWriter, r *http.Request) { "request": fmt.Sprintf("%s %s", r.Method, r.URL.Path), }) - (&httputil.ReverseProxy{ - Director: func(r *http.Request) { - r.URL.Scheme = "http" - r.Host = sessionID + "." + app.serviceName + ":" + app.sidecarPort - r.URL.Host = r.Host - r.Header.Set("X-Forwarded-Selenosis", app.selenosisHost) - logger.Info("proxying session") - }, - ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) { - logger.Errorf("proxying session error: %v", err) - w.WriteHeader(http.StatusBadGateway) - }, - }).ServeHTTP(w, r) + r.URL.Scheme = "http" + r.Host = sessionID + "." + app.serviceName + ":" + app.sidecarPort + r.URL.Host = r.Host + r.Header.Set("X-Forwarded-Selenosis", app.selenosisHost) + + body, err := ioutil.ReadAll(r.Body) + if err != nil { + logger.Errorf("Body readform err: %v", err) + } + + i := 1 + for ; ; i++ { + + rCopy := r.Clone(r.Context()) + r.Body = ioutil.NopCloser(bytes.NewReader(body)) + rCopy.Body = ioutil.NopCloser(bytes.NewReader(body)) + r.ParseForm() + + retryLoop := true + revp := (&httputil.ReverseProxy{ + Director: func(rCopy *http.Request) { + logger.Infof("proxying session -> Body=%v", string(body)) + retryLoop = true + }, + ErrorHandler: func(w http.ResponseWriter, rCopy *http.Request, err error) { + retryLoop = false + logger.Errorf("proxying session error (%d/%d): %v", i, app.sessionRetryCount, err) + if !strings.Contains(err.Error(), "no such host") || i == app.sessionRetryCount { + retryLoop = true + if strings.Contains(err.Error(), "no such host") { + tools.JSONError(w, err.Error(), http.StatusBadRequest) + } else { + tools.JSONError(w, fmt.Sprintf("proxing session error: %v", err.Error()), http.StatusBadRequest) + } + } + }, + }) + + revp.ServeHTTP(w, rCopy) + + if retryLoop || i > app.sessionRetryCount { + break + } + } } -//HandleHubStatus ... +// HandleHubStatus ... func (app *App) HandleHubStatus(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") @@ -244,7 +289,7 @@ func (app *App) HandleHubStatus(w http.ResponseWriter, _ *http.Request) { }) } -//HandleReverseProxy ... +// HandleReverseProxy ... func (app *App) HandleReverseProxy(w http.ResponseWriter, r *http.Request) { sessionID, ok := mux.Vars(r)["sessionId"] if !ok { @@ -281,7 +326,7 @@ func (app *App) HandleReverseProxy(w http.ResponseWriter, r *http.Request) { }).ServeHTTP(w, r) } -//HandleVNC ... +// HandleVNC ... func (app *App) HandleVNC() websocket.Handler { return func(wsconn *websocket.Conn) { defer wsconn.Close() @@ -322,7 +367,7 @@ func (app *App) HandleVNC() websocket.Handler { } } -//HandleLogs ... +// HandleLogs ... func (app *App) HandleLogs() websocket.Handler { return func(wsconn *websocket.Conn) { defer wsconn.Close() @@ -363,7 +408,7 @@ func (app *App) HandleLogs() websocket.Handler { } } -//HandleStatus ... +// HandleStatus ... func (app *App) HandleStatus(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json")