From fdc3a7d100c8a2c79097df798e73e2b308625720 Mon Sep 17 00:00:00 2001 From: Jacob Bednarz Date: Thu, 5 Jan 2023 07:08:39 +1100 Subject: [PATCH] cloudflare: move HTTP request debugging to `httputil` Within the library, we provide a way to output the HTTP interactions that the library makes. We previously used a custom approach and format the output. Even though it works, there are better approaches using `httputil.DumpRequestOut` and `httputil.DumpResponseOut` for dumping the HTTP interactions. Alongside this work, we're adding support for redacting sensitive values in the HTTP interactions. This is useful for both this library and the consumers of this library (like the Terraform Provider) to prevent leaking sensitive information in logs. Closes #1143 --- .changelog/1164.txt | 3 +++ cloudflare.go | 48 ++++++++++++++++++++++++-------------- cloudflare_experimental.go | 45 ++++++++++++++++++++++++----------- 3 files changed, 64 insertions(+), 32 deletions(-) create mode 100644 .changelog/1164.txt diff --git a/.changelog/1164.txt b/.changelog/1164.txt new file mode 100644 index 00000000000..72272d5df35 --- /dev/null +++ b/.changelog/1164.txt @@ -0,0 +1,3 @@ +```release-note:enhancement +cloudflare: automatically redact sensitive values from HTTP interactions +``` diff --git a/cloudflare.go b/cloudflare.go index adf63b94195..5b0c220fcf0 100644 --- a/cloudflare.go +++ b/cloudflare.go @@ -11,7 +11,9 @@ import ( "log" "math" "net/http" + "net/http/httputil" "net/url" + "regexp" "strconv" "strings" "time" @@ -250,20 +252,6 @@ func (api *API) makeRequestWithAuthTypeAndHeadersComplete(ctx context.Context, m return nil, fmt.Errorf("error caused by request rate limiting: %w", err) } - if api.Debug { - if method == http.MethodPost || method == http.MethodPut || method == http.MethodPatch { - buf := &bytes.Buffer{} - tee := io.TeeReader(reqBody, buf) - debugBody, _ := io.ReadAll(tee) - payloadBody, _ := io.ReadAll(buf) - fmt.Printf("cloudflare-go [DEBUG] REQUEST Method:%v URI:%s Headers:%#v Body:%v\n", method, api.BaseURL+uri, headers, string(debugBody)) - // ensure we recreate the io.Reader for use - reqBody = bytes.NewReader(payloadBody) - } else { - fmt.Printf("cloudflare-go [DEBUG] REQUEST Method:%v URI:%s Headers:%#v Body:%v\n", method, api.BaseURL+uri, headers, nil) - } - } - resp, respErr = api.request(ctx, method, uri, reqBody, authType, headers) // short circuit processing on context timeouts @@ -307,10 +295,6 @@ func (api *API) makeRequestWithAuthTypeAndHeadersComplete(ctx context.Context, m return nil, respErr } - if api.Debug { - fmt.Printf("cloudflare-go [DEBUG] RESPONSE StatusCode:%d RayID:%s ContentType:%s Body:%#v\n", resp.StatusCode, resp.Header.Get("cf-ray"), resp.Header.Get("content-type"), string(respBody)) - } - if resp.StatusCode >= http.StatusBadRequest { if strings.HasSuffix(resp.Request.URL.Path, "/filters/validate-expr") { return nil, fmt.Errorf("%s", respBody) @@ -379,6 +363,9 @@ func (api *API) makeRequestWithAuthTypeAndHeadersComplete(ctx context.Context, m // *http.Response, or an error if one occurred. The caller is responsible for // closing the response body. func (api *API) request(ctx context.Context, method, uri string, reqBody io.Reader, authType int, headers http.Header) (*http.Response, error) { + log.SetPrefix(time.Now().Format(time.RFC3339Nano) + " [DEBUG] cloudflare") + log.SetFlags(0) + req, err := http.NewRequestWithContext(ctx, method, api.BaseURL+uri, reqBody) if err != nil { return nil, fmt.Errorf("HTTP request creation failed: %w", err) @@ -408,11 +395,36 @@ func (api *API) request(ctx context.Context, method, uri string, reqBody io.Read req.Header.Set("Content-Type", "application/json") } + if api.Debug { + dump, err := httputil.DumpRequestOut(req, true) + if err != nil { + return nil, err + } + + // Strip out any sensitive information from the request payload. + sensitiveKeys := []string{api.APIKey, api.APIEmail, api.APIToken, api.APIUserServiceKey} + for _, key := range sensitiveKeys { + if key != "" { + valueRegex := regexp.MustCompile(fmt.Sprintf("(?m)%s", key)) + dump = valueRegex.ReplaceAll(dump, []byte("[redacted]")) + } + } + log.Printf("\n%s", string(dump)) + } + resp, err := api.httpClient.Do(req) if err != nil { return nil, fmt.Errorf("HTTP request failed: %w", err) } + if api.Debug { + dump, err := httputil.DumpResponse(resp, true) + if err != nil { + return resp, err + } + log.Printf("\n%s", string(dump)) + } + return resp, nil } diff --git a/cloudflare_experimental.go b/cloudflare_experimental.go index 8856643b5b6..e01c7a60d0f 100644 --- a/cloudflare_experimental.go +++ b/cloudflare_experimental.go @@ -6,8 +6,11 @@ import ( "encoding/json" "fmt" "io" + "log" "net/http" + "net/http/httputil" "net/url" + "regexp" "strings" "sync" "time" @@ -158,6 +161,9 @@ func NewExperimental(config *ClientParams) (*Client, error) { // *http.Response, or an error if one occurred. The caller is responsible for // closing the response body. func (c *Client) request(ctx context.Context, method, uri string, reqBody io.Reader, headers http.Header) (*http.Response, error) { + log.SetPrefix(time.Now().Format(time.RFC3339Nano) + " [DEBUG] cloudflare") + log.SetFlags(0) + req, err := http.NewRequestWithContext(ctx, method, c.BaseURL.String()+uri, reqBody) if err != nil { return nil, fmt.Errorf("HTTP request creation failed: %w", err) @@ -193,11 +199,36 @@ func (c *Client) request(ctx context.Context, method, uri string, reqBody io.Rea req.Header.Set("Content-Type", "application/json") } + if c.Debug { + dump, err := httputil.DumpRequestOut(req, true) + if err != nil { + return nil, err + } + + // Strip out any sensitive information from the request payload. + sensitiveKeys := []string{c.Key, c.Email, c.Token, c.UserServiceKey} + for _, key := range sensitiveKeys { + if key != "" { + valueRegex := regexp.MustCompile(fmt.Sprintf("(?m)%s", key)) + dump = valueRegex.ReplaceAll(dump, []byte("[redacted]")) + } + } + log.Printf("\n%s", string(dump)) + } + resp, err := c.HTTPClient.Do(req) if err != nil { return nil, fmt.Errorf("HTTP request failed: %w", err) } + if c.Debug { + dump, err := httputil.DumpResponse(resp, true) + if err != nil { + return resp, err + } + log.Printf("\n%s", string(dump)) + } + return resp, nil } @@ -224,25 +255,11 @@ func (c *Client) makeRequest(ctx context.Context, method, uri string, params int var respErr error var respBody []byte - if method == http.MethodPost || method == http.MethodPut || method == http.MethodPatch { - buf := &bytes.Buffer{} - tee := io.TeeReader(reqBody, buf) - debugBody, _ := io.ReadAll(tee) - payloadBody, _ := io.ReadAll(buf) - c.Logger.Debugf("REQUEST Method:%v URI:%s Headers:%#v Body:%v\n", method, c.BaseURL.String()+uri, headers, string(debugBody)) - // ensure we recreate the io.Reader for use - reqBody = bytes.NewReader(payloadBody) - } else { - c.Logger.Debugf("REQUEST Method:%v URI:%s Headers:%#v Body:%v\n", method, c.BaseURL.String()+uri, headers, nil) //) - } - resp, respErr = c.request(ctx, method, uri, reqBody, headers) if respErr != nil { return nil, respErr } - c.Logger.Debugf("RESPONSE URI:%s StatusCode:%d Body:%#v RayID:%s\n", c.BaseURL.String()+uri, resp.StatusCode, string(respBody), resp.Header.Get("cf-ray")) - respBody, err = io.ReadAll(resp.Body) resp.Body.Close() if err != nil {