Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

appsec: report http.client_ip only when appsec is enabled #1523

Merged
merged 7 commits into from
Oct 24, 2022
Merged

appsec: report http.client_ip only when appsec is enabled #1523

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
74f9202
internal/appsec: set http client IP before operation start
Hellzy Oct 19, 2022
6cf022a
Merge branch 'main' into francois.mazeau/http-client-ip-rework
Hellzy Oct 20, 2022
e5216e8
internal/appsec: implement IPFromRequest
Hellzy Oct 20, 2022
cf353f6
apm: make http.client_ip an appsec-only feature
Julio-Guerra Oct 24, 2022
d53c7ae
Merge branch 'main' into francois.mazeau/http-client-ip-rework
Julio-Guerra Oct 24, 2022
4925a8a
apm: make http.client_ip an appsec-only feature
Julio-Guerra Oct 24, 2022
04462e0
Fix coding style
Julio-Guerra Oct 24, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion contrib/gin-gonic/gin/appsec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import (
// returns the function to be executed upon finishing the operation
func useAppSec(c *gin.Context, span tracer.Span) func() {
req := c.Request
httpsec.SetAppSecTags(span)
instrumentation.SetAppSecEnabledTags(span)
var params map[string]string
if l := len(c.Params); l > 0 {
params = make(map[string]string, l)
Expand Down
5 changes: 2 additions & 3 deletions contrib/google.golang.org/grpc/appsec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import (
"gopkg.in/DataDog/dd-trace-go.v1/ddtrace"
"gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/dyngo/instrumentation"
"gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/dyngo/instrumentation/grpcsec"
"gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/dyngo/instrumentation/httpsec"

"golang.org/x/net/context"
"google.golang.org/grpc"
Expand All @@ -22,7 +21,7 @@ import (

// UnaryHandler wrapper to use when AppSec is enabled to monitor its execution.
func appsecUnaryHandlerMiddleware(span ddtrace.Span, handler grpc.UnaryHandler) grpc.UnaryHandler {
httpsec.SetAppSecTags(span)
instrumentation.SetAppSecEnabledTags(span)
return func(ctx context.Context, req interface{}) (interface{}, error) {
md, _ := metadata.FromIncomingContext(ctx)
op := grpcsec.StartHandlerOperation(grpcsec.HandlerOperationArgs{Metadata: md}, nil)
Expand All @@ -41,7 +40,7 @@ func appsecUnaryHandlerMiddleware(span ddtrace.Span, handler grpc.UnaryHandler)

// StreamHandler wrapper to use when AppSec is enabled to monitor its execution.
func appsecStreamHandlerMiddleware(span ddtrace.Span, handler grpc.StreamHandler) grpc.StreamHandler {
httpsec.SetAppSecTags(span)
instrumentation.SetAppSecEnabledTags(span)
return func(srv interface{}, stream grpc.ServerStream) error {
md, _ := metadata.FromIncomingContext(stream.Context())
op := grpcsec.StartHandlerOperation(grpcsec.HandlerOperationArgs{Metadata: md}, nil)
Expand Down
8 changes: 0 additions & 8 deletions contrib/internal/httptrace/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,26 +20,18 @@ const (
envQueryStringDisabled = "DD_TRACE_HTTP_URL_QUERY_STRING_DISABLED"
// envQueryStringRegexp is the name of the env var used to specify the regexp to use for query string obfuscation.
envQueryStringRegexp = "DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP"
// envClientIPHeader is the name of the env var used to specify the IP header to be used for client IP collection.
envClientIPHeader = "DD_TRACE_CLIENT_IP_HEADER"
// envClientIPHeader is the name of the env var used to disable client IP tag collection.
envClientIPHeaderDisabled = "DD_TRACE_CLIENT_IP_HEADER_DISABLED"
)

// defaultQueryStringRegexp is the regexp used for query string obfuscation if `envQueryStringRegexp` is empty.
var defaultQueryStringRegexp = regexp.MustCompile("(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:(?:\\s|%20)*(?:=|%3D)[^&]+|(?:\"|%22)(?:\\s|%20)*(?::|%3A)(?:\\s|%20)*(?:\"|%22)(?:%2[^2]|%[^2]|[^\"%])+(?:\"|%22))|bearer(?:\\s|%20)+[a-z0-9\\._\\-]|token(?::|%3A)[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L](?:[\\w=-]|%3D)+\\.ey[I-L](?:[\\w=-]|%3D)+(?:\\.(?:[\\w.+\\/=-]|%3D|%2F|%2B)+)?|[\\-]{5}BEGIN(?:[a-z\\s]|%20)+PRIVATE(?:\\s|%20)KEY[\\-]{5}[^\\-]+[\\-]{5}END(?:[a-z\\s]|%20)+PRIVATE(?:\\s|%20)KEY|ssh-rsa(?:\\s|%20)*(?:[a-z0-9\\/\\.+]|%2F|%5C|%2B){100,}")

type config struct {
queryStringRegexp *regexp.Regexp // specifies the regexp to use for query string obfuscation.
clientIPHeader string // specifies the header to use for IP extraction if client IP tag collection is enabled.
clientIP bool // reports whether the IP should be extracted from the request headers and added to span tags.
queryString bool // reports whether the query string should be included in the URL span tag.
}

func newConfig() config {
c := config{
clientIPHeader: os.Getenv(envClientIPHeader),
clientIP: !internal.BoolEnv(envClientIPHeaderDisabled, false),
queryString: !internal.BoolEnv(envQueryStringDisabled, false),
queryStringRegexp: defaultQueryStringRegexp,
}
Expand Down
24 changes: 4 additions & 20 deletions contrib/internal/httptrace/config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import (

func TestConfig(t *testing.T) {
defaultCfg := config{
clientIP: true,
queryString: true,
queryStringRegexp: defaultQueryStringRegexp,
}
Expand All @@ -30,25 +29,15 @@ func TestConfig(t *testing.T) {
{
name: "bad-values",
env: map[string]string{
envQueryStringDisabled: "invalid",
envClientIPHeaderDisabled: "invalid",
envQueryStringRegexp: "+",
envQueryStringDisabled: "invalid",
envQueryStringRegexp: "+",
},
cfg: defaultCfg,
},
{
name: "disable-query",
env: map[string]string{envQueryStringDisabled: "true"},
cfg: config{
clientIP: true,
queryStringRegexp: defaultQueryStringRegexp,
},
},
{
name: "disable-ip",
env: map[string]string{envClientIPHeaderDisabled: "true"},
cfg: config{
queryString: true,
queryStringRegexp: defaultQueryStringRegexp,
},
},
Expand All @@ -57,7 +46,6 @@ func TestConfig(t *testing.T) {
env: map[string]string{envQueryStringRegexp: ""},
cfg: config{
queryString: true,
clientIP: true,
},
},
} {
Expand All @@ -69,18 +57,14 @@ func TestConfig(t *testing.T) {
c := newConfig()
require.Equal(t, tc.cfg.queryStringRegexp, c.queryStringRegexp)
require.Equal(t, tc.cfg.queryString, c.queryString)
require.Equal(t, tc.cfg.clientIPHeader, c.clientIPHeader)
require.Equal(t, tc.cfg.clientIP, c.clientIP)
})
}
}

func cleanEnv() func() {
env := map[string]string{
envQueryStringDisabled: os.Getenv(envQueryStringDisabled),
envQueryStringRegexp: os.Getenv(envQueryStringRegexp),
envClientIPHeaderDisabled: os.Getenv(envClientIPHeaderDisabled),
envClientIPHeader: os.Getenv(envClientIPHeader),
envQueryStringDisabled: os.Getenv(envQueryStringDisabled),
envQueryStringRegexp: os.Getenv(envQueryStringRegexp),
}
for k := range env {
os.Unsetenv(k)
Expand Down
99 changes: 1 addition & 98 deletions contrib/internal/httptrace/httptrace.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ package httptrace
import (
"context"
"fmt"
"net"
"net/http"
"strconv"
"strings"
Expand All @@ -20,27 +19,7 @@ import (
"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
)

var (
ipv6SpecialNetworks = []*netaddrIPPrefix{
ippref("fec0::/10"), // site local
}
defaultIPHeaders = []string{
"x-forwarded-for",
"x-real-ip",
"x-client-ip",
"x-forwarded",
"x-cluster-client-ip",
"forwarded-for",
"forwarded",
"via",
"true-client-ip",
}
cfg = newConfig()
)

// multipleIPHeaders sets the multiple ip header tag used internally to tell the backend an error occurred when
// retrieving an HTTP request client IP.
const multipleIPHeaders = "_dd.multiple-ip-headers"
var cfg = newConfig()

// StartRequestSpan starts an HTTP request span with the standard list of HTTP request span tags (http.method, http.url,
// http.useragent). Any further span start option can be added with opts.
Expand All @@ -59,9 +38,6 @@ func StartRequestSpan(r *http.Request, opts ...ddtrace.StartSpanOption) (tracer.
tracer.Tag("http.host", r.Host),
}, opts...)
}
if cfg.clientIP {
opts = append(genClientIPSpanTags(r), opts...)
}
if spanctx, err := tracer.Extract(tracer.HTTPHeadersCarrier(r.Header)); err == nil {
opts = append(opts, tracer.ChildOf(spanctx))
}
Expand All @@ -84,79 +60,6 @@ func FinishRequestSpan(s tracer.Span, status int, opts ...tracer.FinishOption) {
s.Finish(opts...)
}

// ippref returns the IP network from an IP address string s. If not possible, it returns nil.
func ippref(s string) *netaddrIPPrefix {
if prefix, err := netaddrParseIPPrefix(s); err == nil {
return &prefix
}
return nil
}

// genClientIPSpanTags generates the client IP related tags that need to be added to the span.
// See https://docs.datadoghq.com/tracing/configure_data_security#configuring-a-client-ip-header for more information.
func genClientIPSpanTags(r *http.Request) []ddtrace.StartSpanOption {
ipHeaders := defaultIPHeaders
if len(cfg.clientIPHeader) > 0 {
ipHeaders = []string{cfg.clientIPHeader}
}
var headers []string
var ips []string
var opts []ddtrace.StartSpanOption
for _, hdr := range ipHeaders {
if v := r.Header.Get(hdr); v != "" {
headers = append(headers, hdr)
ips = append(ips, v)
}
}
if len(ips) == 0 {
if remoteIP := parseIP(r.RemoteAddr); remoteIP.IsValid() && isGlobal(remoteIP) {
opts = append(opts, tracer.Tag(ext.HTTPClientIP, remoteIP.String()))
}
} else if len(ips) == 1 {
for _, ipstr := range strings.Split(ips[0], ",") {
ip := parseIP(strings.TrimSpace(ipstr))
if ip.IsValid() && isGlobal(ip) {
opts = append(opts, tracer.Tag(ext.HTTPClientIP, ip.String()))
break
}
}
} else {
for i := range ips {
opts = append(opts, tracer.Tag(ext.HTTPRequestHeaders+"."+headers[i], ips[i]))
}
opts = append(opts, tracer.Tag(multipleIPHeaders, strings.Join(headers, ",")))
}
return opts
}

func parseIP(s string) netaddrIP {
if ip, err := netaddrParseIP(s); err == nil {
return ip
}
if h, _, err := net.SplitHostPort(s); err == nil {
if ip, err := netaddrParseIP(h); err == nil {
return ip
}
}
return netaddrIP{}
}

func isGlobal(ip netaddrIP) bool {
// IsPrivate also checks for ipv6 ULA.
// We care to check for these addresses are not considered public, hence not global.
// See https://www.rfc-editor.org/rfc/rfc4193.txt for more details.
isGlobal := !ip.IsPrivate() && !ip.IsLoopback() && !ip.IsLinkLocalUnicast()
if !isGlobal || !ip.Is6() {
return isGlobal
}
for _, n := range ipv6SpecialNetworks {
if n.Contains(ip) {
return false
}
}
return isGlobal
}

// urlFromRequest returns the full URL from the HTTP request. If query params are collected, they are obfuscated granted
// obfuscation is not disabled by the user (through DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP)
// See https://docs.datadoghq.com/tracing/configure_data_security#redacting-the-query-in-the-url for more information.
Expand Down