Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(engine): add trustedproxies and remoteIP #2632

Merged
merged 24 commits into from Apr 6, 2021
Merged
Show file tree
Hide file tree
Changes from 22 commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
de0f9eb
X-Forwarded-For handling is unsafe
sorenisanerd Aug 20, 2020
b343e7e
Merge branch 'master' into sorenh/issue2473
appleboy Jan 4, 2021
b7a35bf
Merge branch 'master' into sorenh/issue2473
thinkerou Jan 11, 2021
d0bf406
Merge branch 'master' into sorenh/issue2473
appleboy Jan 23, 2021
31a2afa
Merge branch 'sorenh/issue2473' of https://github.com/sorenh/gin into…
manucorporat Feb 7, 2021
c9ea8ec
refactor(): fix client Ip vulnerability
manucorporat Feb 7, 2021
39b372f
chore(engine): remove unused lookupHost
javierprovecho Feb 8, 2021
e14a43c
fix(engine): wrong syntax/behaviour at prepareCIDR
javierprovecho Feb 8, 2021
55ad88a
refactor move logic to remoteIP()
manucorporat Feb 8, 2021
6f562ea
add docs
manucorporat Feb 8, 2021
ffe7ac0
🐈
manucorporat Feb 8, 2021
2d426b6
add explanation 🦀
manucorporat Feb 8, 2021
fc99953
fix 🐨
manucorporat Feb 8, 2021
15f576c
docs(readme): remove old reference to hostnames
javierprovecho Feb 8, 2021
ba157c9
Merge branch 'master' into fix-client-ip-cve
manucorporat Feb 9, 2021
a598663
docs(engine): explain new gin.Engine properties
javierprovecho Feb 9, 2021
7e649c3
feat(engine): add ipv6 support to TrustedProxies
javierprovecho Feb 10, 2021
0876678
fix error
thinkerou Mar 27, 2021
56fbadc
Merge branch 'master' into fix-client-ip-cve
thinkerou Mar 27, 2021
feaee20
add unit test
thinkerou Mar 27, 2021
2d7fc06
add unit test
thinkerou Mar 27, 2021
3483d2c
remove nil statement
thinkerou Mar 27, 2021
65711ec
Merge branch 'master' into fix-client-ip-cve
appleboy Apr 4, 2021
9018d58
Merge branch 'master' into fix-client-ip-cve
appleboy Apr 6, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
33 changes: 33 additions & 0 deletions README.md
Expand Up @@ -2117,6 +2117,39 @@ func main() {
}
```

## Don't trust all proxies

Gin lets you specify which headers to hold the real client IP (if any),
as well as specifying which proxies (or direct clients) you trust to
specify one of these headers.

The `TrustedProxies` slice on your `gin.Engine` specifes network addresses or
network CIDRs from where clients which their request headers related to client
IP can be trusted. They can be IPv4 addresses, IPv4 CIDRs, IPv6 addresses or
IPv6 CIDRs.

```go
import (
"fmt"

"github.com/gin-gonic/gin"
)

func main() {

router := gin.Default()
router.TrustedProxies = []string{"192.168.1.2"}

router.GET("/", func(c *gin.Context) {
// If the client is 192.168.1.2, use the X-Forwarded-For
// header to deduce the original client IP from the trust-
// worthy parts of that header.
// Otherwise, simply return the direct client IP
fmt.Printf("ClientIP: %s\n", c.ClientIP())
})
router.Run()
}
manucorporat marked this conversation as resolved.
Show resolved Hide resolved
```

## Testing

Expand Down
82 changes: 66 additions & 16 deletions context.go
Expand Up @@ -725,32 +725,82 @@ func (c *Context) ShouldBindBodyWith(obj interface{}, bb binding.BindingBody) (e
return bb.BindBody(body, obj)
}

// ClientIP implements a best effort algorithm to return the real client IP, it parses
// X-Real-IP and X-Forwarded-For in order to work properly with reverse-proxies such us: nginx or haproxy.
// Use X-Forwarded-For before X-Real-Ip as nginx uses X-Real-Ip with the proxy's IP.
// ClientIP implements a best effort algorithm to return the real client IP.
// It called c.RemoteIP() under the hood, to check if the remote IP is a trusted proxy or not.
// If it's it will then try to parse the headers defined in Engine.RemoteIPHeaders (defaulting to [X-Forwarded-For, X-Real-Ip]).
// If the headers are nots syntactically valid OR the remote IP does not correspong to a trusted proxy,
// the remote IP (coming form Request.RemoteAddr) is returned.
func (c *Context) ClientIP() string {
if c.engine.ForwardedByClientIP {
clientIP := c.requestHeader("X-Forwarded-For")
clientIP = strings.TrimSpace(strings.Split(clientIP, ",")[0])
if clientIP == "" {
clientIP = strings.TrimSpace(c.requestHeader("X-Real-Ip"))
if c.engine.AppEngine {
if addr := c.requestHeader("X-Appengine-Remote-Addr"); addr != "" {
return addr
}
if clientIP != "" {
return clientIP
}

remoteIP, trusted := c.RemoteIP()
if remoteIP == nil {
return ""
}

if trusted && c.engine.ForwardedByClientIP && c.engine.RemoteIPHeaders != nil {
for _, headerName := range c.engine.RemoteIPHeaders {
ip, valid := validateHeader(c.requestHeader(headerName))
if valid {
return ip
}
}
}
return remoteIP.String()
}

if c.engine.AppEngine {
if addr := c.requestHeader("X-Appengine-Remote-Addr"); addr != "" {
return addr
// RemoteIP parses the IP from Request.RemoteAddr, normalizes and returns the IP (without the port).
// It also checks if the remoteIP is a trusted proxy or not.
// In order to perform this validation, it will see if the IP is contained within at least one of the CIDR blocks
// defined in Engine.TrustedProxies
func (c *Context) RemoteIP() (net.IP, bool) {
ip, _, err := net.SplitHostPort(strings.TrimSpace(c.Request.RemoteAddr))
if err != nil {
return nil, false
}
remoteIP := net.ParseIP(ip)
if remoteIP == nil {
return nil, false
}

trustedCIDRs, _ := c.engine.prepareTrustedCIDRs()
c.engine.trustedCIDRs = trustedCIDRs
if c.engine.trustedCIDRs != nil {
for _, cidr := range c.engine.trustedCIDRs {
if cidr.Contains(remoteIP) {
return remoteIP, true
}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

			if !cidr.Contains(remoteIP) {
                             continue
                         }
			for _, headerName := range c.engine.RemoteIPHeaders {
				ip, valid := validateHeader(c.requestHeader(headerName))
				if valid {
					return ip
				}
			}

suggestion, thanks!

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

refactored it a little bit, review again... now thee logic is better separated between validating the trusted proxy and parsing the header, also a nw AP

ip, trusted = c.RemoteIP()

allows to even implement your own logic, or trust othe headers that might not be even related with IP!

Copy link
Contributor

@agmt agmt Aug 24, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you expect that HTTP proxy running on c.RemoteIP() resets X-Forwarded-For? Because if it appends, then we can't inherit trustiness of c.RemoteIP() to all other proxies.

}
}

if ip, _, err := net.SplitHostPort(strings.TrimSpace(c.Request.RemoteAddr)); err == nil {
return ip
return remoteIP, false
}

func validateHeader(header string) (clientIP string, valid bool) {
if header == "" {
return "", false
}
items := strings.Split(header, ",")
for i, ipStr := range items {
ipStr = strings.TrimSpace(ipStr)
ip := net.ParseIP(ipStr)
if ip == nil {
return "", false
}

return ""
// We need to return the first IP in the list, but,
// we should not early return since we need to validate that
// the rest of the header is syntactically valid
if i == 0 {
clientIP = ipStr
valid = true
}
}
return
}

// ContentType returns the Content-Type header of the request.
Expand Down
84 changes: 80 additions & 4 deletions context_test.go
Expand Up @@ -1392,11 +1392,10 @@ func TestContextClientIP(t *testing.T) {
c, _ := CreateTestContext(httptest.NewRecorder())
c.Request, _ = http.NewRequest("POST", "/", nil)

c.Request.Header.Set("X-Real-IP", " 10.10.10.10 ")
c.Request.Header.Set("X-Forwarded-For", " 20.20.20.20, 30.30.30.30")
c.Request.Header.Set("X-Appengine-Remote-Addr", "50.50.50.50")
c.Request.RemoteAddr = " 40.40.40.40:42123 "
resetContextForClientIPTests(c)

// Legacy tests (validating that the defaults don't break the
// (insecure!) old behaviour)
assert.Equal(t, "20.20.20.20", c.ClientIP())

c.Request.Header.Del("X-Forwarded-For")
Expand All @@ -1416,6 +1415,74 @@ func TestContextClientIP(t *testing.T) {
// no port
c.Request.RemoteAddr = "50.50.50.50"
assert.Empty(t, c.ClientIP())

// Tests exercising the TrustedProxies functionality
resetContextForClientIPTests(c)

// No trusted proxies
c.engine.TrustedProxies = []string{}
c.engine.RemoteIPHeaders = []string{"X-Forwarded-For"}
assert.Equal(t, "40.40.40.40", c.ClientIP())

// Last proxy is trusted, but the RemoteAddr is not
c.engine.TrustedProxies = []string{"30.30.30.30"}
assert.Equal(t, "40.40.40.40", c.ClientIP())

// Only trust RemoteAddr
c.engine.TrustedProxies = []string{"40.40.40.40"}
assert.Equal(t, "20.20.20.20", c.ClientIP())
Copy link
Contributor

@agmt agmt Aug 24, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we trust proxy 40.40.40.40, but not trust 30.30.30.30 (proxy it is or not), then ClientIP should be 30.30.30.30 as 20.20.20.20 was set by somebody untrusted.
Please do not forget that X-Forwarded-For is appended, so it should be processed right-to-left:

right-most IP address is the IP address of the most recent proxy and the left-most IP address is the IP address of the originating client.


// All steps are trusted
c.engine.TrustedProxies = []string{"40.40.40.40", "30.30.30.30", "20.20.20.20"}
assert.Equal(t, "20.20.20.20", c.ClientIP())

// Use CIDR
c.engine.TrustedProxies = []string{"40.40.25.25/16", "30.30.30.30"}
assert.Equal(t, "20.20.20.20", c.ClientIP())

// Use hostname that resolves to all the proxies
c.engine.TrustedProxies = []string{"foo"}
assert.Equal(t, "40.40.40.40", c.ClientIP())

// Use hostname that returns an error
c.engine.TrustedProxies = []string{"bar"}
assert.Equal(t, "40.40.40.40", c.ClientIP())

// X-Forwarded-For has a non-IP element
c.engine.TrustedProxies = []string{"40.40.40.40"}
c.Request.Header.Set("X-Forwarded-For", " blah ")
assert.Equal(t, "40.40.40.40", c.ClientIP())

// Result from LookupHost has non-IP element. This should never
// happen, but we should test it to make sure we handle it
// gracefully.
c.engine.TrustedProxies = []string{"baz"}
c.Request.Header.Set("X-Forwarded-For", " 30.30.30.30 ")
assert.Equal(t, "40.40.40.40", c.ClientIP())

c.engine.TrustedProxies = []string{"40.40.40.40"}
c.Request.Header.Del("X-Forwarded-For")
c.engine.RemoteIPHeaders = []string{"X-Forwarded-For", "X-Real-IP"}
assert.Equal(t, "10.10.10.10", c.ClientIP())

c.engine.RemoteIPHeaders = []string{}
c.engine.AppEngine = true
assert.Equal(t, "50.50.50.50", c.ClientIP())

c.Request.Header.Del("X-Appengine-Remote-Addr")
assert.Equal(t, "40.40.40.40", c.ClientIP())

// no port
c.Request.RemoteAddr = "50.50.50.50"
assert.Empty(t, c.ClientIP())
}

func resetContextForClientIPTests(c *Context) {
c.Request.Header.Set("X-Real-IP", " 10.10.10.10 ")
c.Request.Header.Set("X-Forwarded-For", " 20.20.20.20, 30.30.30.30")
c.Request.Header.Set("X-Appengine-Remote-Addr", "50.50.50.50")
c.Request.RemoteAddr = " 40.40.40.40:42123 "
c.engine.AppEngine = false
}

func TestContextContentType(t *testing.T) {
Expand Down Expand Up @@ -1960,3 +2027,12 @@ func TestContextWithKeysMutex(t *testing.T) {
assert.Nil(t, value)
assert.False(t, err)
}

func TestRemoteIPFail(t *testing.T) {
c, _ := CreateTestContext(httptest.NewRecorder())
c.Request, _ = http.NewRequest("POST", "/", nil)
c.Request.RemoteAddr = "[:::]:80"
ip, trust := c.RemoteIP()
assert.Nil(t, ip)
assert.False(t, trust)
}
73 changes: 71 additions & 2 deletions gin.go
Expand Up @@ -11,6 +11,7 @@ import (
"net/http"
"os"
"path"
"strings"
"sync"

"github.com/gin-gonic/gin/internal/bytesconv"
Expand Down Expand Up @@ -81,9 +82,26 @@ type Engine struct {
// If no other Method is allowed, the request is delegated to the NotFound
// handler.
HandleMethodNotAllowed bool
ForwardedByClientIP bool

// #726 #755 If enabled, it will thrust some headers starting with
// If enabled, client IP will be parsed from the request's headers that
// match those stored at `(*gin.Engine).RemoteIPHeaders`. If no IP was
// fetched, it falls back to the IP obtained from
// `(*gin.Context).Request.RemoteAddr`.
ForwardedByClientIP bool

// List of headers used to obtain the client IP when
// `(*gin.Engine).ForwardedByClientIP` is `true` and
// `(*gin.Context).Request.RemoteAddr` is matched by at least one of the
// network origins of `(*gin.Engine).TrustedProxies`.
RemoteIPHeaders []string

// List of network origins (IPv4 addresses, IPv4 CIDRs, IPv6 addresses or
// IPv6 CIDRs) from which to trust request's headers that contain
// alternative client IP when `(*gin.Engine).ForwardedByClientIP` is
// `true`.
TrustedProxies []string

// #726 #755 If enabled, it will trust some headers starting with
// 'X-AppEngine...' for better integration with that PaaS.
AppEngine bool

Expand Down Expand Up @@ -114,6 +132,7 @@ type Engine struct {
pool sync.Pool
trees methodTrees
maxParams uint16
trustedCIDRs []*net.IPNet
}

var _ IRouter = &Engine{}
Expand All @@ -139,6 +158,8 @@ func New() *Engine {
RedirectFixedPath: false,
HandleMethodNotAllowed: false,
ForwardedByClientIP: true,
RemoteIPHeaders: []string{"X-Forwarded-For", "X-Real-IP"},
TrustedProxies: []string{"0.0.0.0/0"},
AppEngine: defaultAppEngine,
UseRawPath: false,
RemoveExtraSlash: false,
Expand Down Expand Up @@ -305,12 +326,60 @@ func iterate(path, method string, routes RoutesInfo, root *node) RoutesInfo {
func (engine *Engine) Run(addr ...string) (err error) {
defer func() { debugPrintError(err) }()

trustedCIDRs, err := engine.prepareTrustedCIDRs()
if err != nil {
return err
}
engine.trustedCIDRs = trustedCIDRs
address := resolveAddress(addr)
debugPrint("Listening and serving HTTP on %s\n", address)
err = http.ListenAndServe(address, engine)
return
}

func (engine *Engine) prepareTrustedCIDRs() ([]*net.IPNet, error) {
if engine.TrustedProxies == nil {
return nil, nil
}

cidr := make([]*net.IPNet, 0, len(engine.TrustedProxies))
for _, trustedProxy := range engine.TrustedProxies {
if !strings.Contains(trustedProxy, "/") {
ip := parseIP(trustedProxy)
if ip == nil {
return cidr, &net.ParseError{Type: "IP address", Text: trustedProxy}
}

switch len(ip) {
case net.IPv4len:
trustedProxy += "/32"
case net.IPv6len:
trustedProxy += "/128"
}
}
_, cidrNet, err := net.ParseCIDR(trustedProxy)
if err != nil {
return cidr, err
}
cidr = append(cidr, cidrNet)
}
return cidr, nil
}

// parseIP parse a string representation of an IP and returns a net.IP with the
// minimum byte representation or nil if input is invalid.
func parseIP(ip string) net.IP {
parsedIP := net.ParseIP(ip)

if ipv4 := parsedIP.To4(); ipv4 != nil {
// return ip in a 4-byte representation
return ipv4
}

// return ip in a 16-byte representation or nil
return parsedIP
}

// RunTLS attaches the router to a http.Server and starts listening and serving HTTPS (secure) requests.
// It is a shortcut for http.ListenAndServeTLS(addr, certFile, keyFile, router)
// Note: this method will block the calling goroutine indefinitely unless an error happens.
Expand Down
7 changes: 7 additions & 0 deletions gin_integration_test.go
Expand Up @@ -55,6 +55,13 @@ func TestRunEmpty(t *testing.T) {
testRequest(t, "http://localhost:8080/example")
}

func TestTrustedCIDRsForRun(t *testing.T) {
os.Setenv("PORT", "")
router := New()
router.TrustedProxies = []string{"hello/world"}
assert.Error(t, router.Run(":8080"))
}

func TestRunTLS(t *testing.T) {
router := New()
go func() {
Expand Down