Permalink
Browse files

Added shared transports for Go HTTP clients

  • Loading branch information...
zweizeichen committed Sep 28, 2017
1 parent 4f8b27b commit 62bc7d84f14de871226d7af8ee053147d6c78696
Showing with 110 additions and 0 deletions.
  1. +84 −0 go/lib/transport/esiTransport.go
  2. +26 −0 go/lib/transport/transport.go
@@ -0,0 +1,84 @@
package transport
import (
"errors"
"net/http"
"strconv"
"sync"
"time"
"github.com/sirupsen/logrus"
)
// ESITransport is a custom transport for rate-limiting requests to ESI
type ESITransport struct {
userAgent string
timeout time.Duration
blockedUntil time.Time
mutex sync.RWMutex
}
// NewESITransport generates a new custom transport
func NewESITransport(userAgent string, timeout time.Duration) *ESITransport {
return &ESITransport{
userAgent: userAgent,
timeout: timeout,
blockedUntil: time.Now(),
}
}
// RoundTrip uses a default transport which blocks when the client is being blocked by ESI or we're close to the limit.
// Waiting requests will pile-up! Having backpressure in place should be considered.
func (transport *ESITransport) RoundTrip(request *http.Request) (*http.Response, error) {
transport.mutex.RLock()
unlockTime := transport.blockedUntil
timeoutTime := time.Now().Add(transport.timeout)
transport.mutex.RUnlock()
if unlockTime.After(time.Now()) {
// Check if sleeping would exceed timeout, if yes, cancel instantly
if unlockTime.After(timeoutTime) {
return nil, errors.New("skipped request, delay would exceed timeout")
}
// Sleep until unblock if we're blocked by ESI
time.Sleep(unlockTime.Sub(time.Now()))
}
request.Header.Set("User-Agent", transport.userAgent)
response, err := http.DefaultTransport.RoundTrip(request)
if err != nil {
return response, err
}
if response != nil {
// Try to parse the remaining errors header
remainingErrors, err := strconv.ParseInt(response.Header.Get("X-ESI-Error-Limit-Remain"), 10, 64)
if err != nil {
// Set to default value above limit
remainingErrors = 100
}
// We're blocked or at least close, defer/skip requests until reset of error window
if remainingErrors <= 3 || response.StatusCode == 420 || response.Header.Get("X-ESI-Error-Limited") != "" {
reset := response.Header.Get("X-ESI-Error-Limit-Reset")
logrus.WithField("reset_in", reset).Warn("Too many errors when calling ESI, waiting until error window resets!")
if reset != "" {
windowEndsInSeconds, err := strconv.ParseInt(reset, 10, 64)
if err != nil {
return response, err
}
transport.mutex.Lock()
transport.blockedUntil = time.Now().Add(time.Duration(windowEndsInSeconds * 1e9))
transport.mutex.Unlock()
}
}
}
return response, err
}
@@ -0,0 +1,26 @@
package transport
import (
"net/http"
)
// Transport is a custom transport for Element43 which simply sets the user agent
type Transport struct {
userAgent string
}
// NewTransport generates a new custom transport
func NewTransport(userAgent string) *Transport {
return &Transport{
userAgent: userAgent,
}
}
// RoundTrip automatically sets the user agent
func (transport *Transport) RoundTrip(request *http.Request) (*http.Response, error) {
request.Header.Set("User-Agent", transport.userAgent)
response, err := http.DefaultTransport.RoundTrip(request)
return response, err
}

0 comments on commit 62bc7d8

Please sign in to comment.