Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix(httpext): remove user credentials from metric tags #1132

Merged
merged 3 commits into from Sep 26, 2019
Merged

fix(httpext): remove user credentials from metric tags #1132

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8d61e40
fix(httpext): remove user credentials from metric tags
Aug 29, 2019
6d6291c
wip(httpext): refactor cleanURL
Aug 30, 2019
ed6e0e7
perf(httpext): more efficient credential masking implementation
Sep 25, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
21 changes: 14 additions & 7 deletions js/modules/k6/http/request_test.go
View file
Open in desktop
Expand Up @@ -751,17 +751,19 @@ func TestRequestAndBatch(t *testing.T) {
t.Run("auth", func(t *testing.T) {
t.Run("basic", func(t *testing.T) {
url := sr("http://bob:pass@HTTPBIN_IP:HTTPBIN_PORT/basic-auth/bob/pass")
urlExpected := sr("http://****:****@HTTPBIN_IP:HTTPBIN_PORT/basic-auth/bob/pass")

_, err := common.RunString(rt, fmt.Sprintf(`
let res = http.request("GET", "%s", null, {});
if (res.status != 200) { throw new Error("wrong status: " + res.status); }
`, url))
assert.NoError(t, err)
assertRequestMetricsEmitted(t, stats.GetBufferedSamples(samples), "GET", url, "", 200, "")
assertRequestMetricsEmitted(t, stats.GetBufferedSamples(samples), "GET", urlExpected, "", 200, "")
})
t.Run("digest", func(t *testing.T) {
t.Run("success", func(t *testing.T) {
url := sr("http://bob:pass@HTTPBIN_IP:HTTPBIN_PORT/digest-auth/auth/bob/pass")
urlExpected := sr("http://****:****@HTTPBIN_IP:HTTPBIN_PORT/digest-auth/auth/bob/pass")

_, err := common.RunString(rt, fmt.Sprintf(`
let res = http.request("GET", "%s", null, { auth: "digest" });
Expand All @@ -771,8 +773,10 @@ func TestRequestAndBatch(t *testing.T) {
assert.NoError(t, err)

sampleContainers := stats.GetBufferedSamples(samples)
assertRequestMetricsEmitted(t, sampleContainers[0:1], "GET", sr("HTTPBIN_IP_URL/digest-auth/auth/bob/pass"), url, 401, "")
assertRequestMetricsEmitted(t, sampleContainers[1:2], "GET", sr("HTTPBIN_IP_URL/digest-auth/auth/bob/pass"), url, 200, "")
assertRequestMetricsEmitted(t, sampleContainers[0:1], "GET",
sr("HTTPBIN_IP_URL/digest-auth/auth/bob/pass"), urlExpected, 401, "")
assertRequestMetricsEmitted(t, sampleContainers[1:2], "GET",
sr("HTTPBIN_IP_URL/digest-auth/auth/bob/pass"), urlExpected, 200, "")
})
t.Run("failure", func(t *testing.T) {
url := sr("http://bob:pass@HTTPBIN_IP:HTTPBIN_PORT/digest-auth/failure")
Expand Down Expand Up @@ -1911,7 +1915,6 @@ func TestDigestAuthWithBody(t *testing.T) {
httpbin.New().DigestAuth(w, r) // this doesn't read the body
}))

// TODO: fix, the metric tags shouldn't have credentials (https://github.com/loadimpact/k6/issues/1103)
urlWithCreds := tb.Replacer.Replace(
"http://testuser:testpwd@HTTPBIN_IP:HTTPBIN_PORT/digest-auth-with-post/auth/testuser/testpwd",
)
Expand All @@ -1923,8 +1926,12 @@ func TestDigestAuthWithBody(t *testing.T) {
`, urlWithCreds))
require.NoError(t, err)

expectedURL := tb.Replacer.Replace("HTTPBIN_IP_URL/digest-auth-with-post/auth/testuser/testpwd")
expectedURL := tb.Replacer.Replace(
"http://HTTPBIN_IP:HTTPBIN_PORT/digest-auth-with-post/auth/testuser/testpwd")
expectedName := tb.Replacer.Replace(
"http://****:****@HTTPBIN_IP:HTTPBIN_PORT/digest-auth-with-post/auth/testuser/testpwd")

sampleContainers := stats.GetBufferedSamples(samples)
assertRequestMetricsEmitted(t, sampleContainers[0:1], "POST", expectedURL, urlWithCreds, 401, "")
assertRequestMetricsEmitted(t, sampleContainers[1:2], "POST", expectedURL, urlWithCreds, 200, "")
assertRequestMetricsEmitted(t, sampleContainers[0:1], "POST", expectedURL, expectedName, 401, "")
assertRequestMetricsEmitted(t, sampleContainers[1:2], "POST", expectedURL, expectedName, 200, "")
}
37 changes: 32 additions & 5 deletions lib/netext/httpext/request.go
View file
Open in desktop
Expand Up @@ -49,16 +49,43 @@ type HTTPRequestCookie struct {

// A URL wraps net.URL, and preserves the template (if any) the URL was constructed from.
type URL struct {
u *url.URL
Name string // http://example.com/thing/${}/
URL string // http://example.com/thing/1234/
u *url.URL
Name string // http://example.com/thing/${}/
URL string // http://example.com/thing/1234/
CleanURL string // URL with masked user credentials, used for output
}

// NewURL returns a new URL for the provided url and name. The error is returned if the url provided
// can't be parsed
func NewURL(urlString, name string) (URL, error) {
u, err := url.Parse(urlString)
return URL{u: u, Name: name, URL: urlString}, err
newURL := URL{u: u, Name: name, URL: urlString}
newURL.CleanURL = newURL.Clean()
if urlString == name {
newURL.Name = newURL.CleanURL
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This handles this use case.

}
return newURL, err
}

// Clean returns an output-safe representation of URL
func (u URL) Clean() string {
if u.CleanURL != "" {
return u.CleanURL
}

out := u.URL

if u.u != nil && u.u.User != nil {
schemeIndex := strings.Index(out, "://")
atIndex := strings.Index(out, "@")
if _, passwordOk := u.u.User.Password(); passwordOk {
out = out[:schemeIndex+3] + "****:****" + out[atIndex:]
} else {
out = out[:schemeIndex+3] + "****" + out[atIndex:]
}
}

return out
}

// GetURL returns the internal url.URL
Expand Down Expand Up @@ -212,7 +239,7 @@ func MakeRequest(ctx context.Context, preq *ParsedHTTPRequest) (*Response, error
tags["method"] = preq.Req.Method
}
if state.Options.SystemTags["url"] {
tags["url"] = preq.URL.URL
tags["url"] = preq.URL.Clean()
}

// Only set the name system tag if the user didn't explicitly set it beforehand
Expand Down
29 changes: 29 additions & 0 deletions lib/netext/httpext/request_test.go
View file
Open in desktop
Expand Up @@ -6,6 +6,7 @@ import (
"io"
"io/ioutil"
"net/http"
"net/url"
"testing"

"github.com/pkg/errors"
Expand Down Expand Up @@ -82,3 +83,31 @@ func TestMakeRequestError(t *testing.T) {
require.Equal(t, err.Error(), "unknown compressionType CompressionType(13)")
})
}

func TestURL(t *testing.T) {
t.Run("Clean", func(t *testing.T) {
testCases := []struct {
url string
expected string
}{
{"https://example.com/", "https://example.com/"},
{"https://example.com/${}", "https://example.com/${}"},
{"https://user@example.com/", "https://****@example.com/"},
{"https://user:pass@example.com/", "https://****:****@example.com/"},
{"https://user:pass@example.com/path?a=1&b=2", "https://****:****@example.com/path?a=1&b=2"},
{"https://user:pass@example.com/${}/${}", "https://****:****@example.com/${}/${}"},
{"@malformed/url", "@malformed/url"},
{"not a url", "not a url"},
}

for _, tc := range testCases {
tc := tc
t.Run(tc.url, func(t *testing.T) {
u, err := url.Parse(tc.url)
require.NoError(t, err)
ut := URL{u: u, URL: tc.url}
require.Equal(t, tc.expected, ut.Clean())
})
}
})
}
3 changes: 2 additions & 1 deletion lib/netext/httpext/transport.go
View file
Open in desktop
Expand Up @@ -113,7 +113,8 @@ func (t *transport) measureAndEmitMetrics(unfReq *unfinishedRequest) *finishedRe
}
} else {
if enabledTags["url"] {
tags["url"] = unfReq.request.URL.String()
u := URL{u: unfReq.request.URL, URL: unfReq.request.URL.String()}
tags["url"] = u.Clean()
}
if enabledTags["status"] {
tags["status"] = strconv.Itoa(unfReq.response.StatusCode)
Expand Down