feat: put object tests and better logging (#38)

api/api.go

@@ -71,7 +71,10 @@ func (a *Api) routeBase(w http.ResponseWriter, req *http.Request) {
 	if err != nil {
 		a.InternalError(sess.Logger(), w, err)
 		return
-	} else if failover {
+	}
+
+	sess.WithLogger(sess.Logger().With(zap.Int("statusCode", resp.StatusCode)))
+	if failover {
 		sess.WithLogger(sess.Logger().With(zap.Bool("failover", true)))
 	}
 
@@ -82,7 +85,10 @@ func (a *Api) routeBase(w http.ResponseWriter, req *http.Request) {
 	}
 
 	if !boltrouter.StatusCodeIs2xx(resp.StatusCode) {
-		sess.Logger().Warn("Status code is not 2xx in s3 response", zap.Int("statusCode", resp.StatusCode))
+		body := boltrouter.CopyRespBody(resp)
+		b, _ := io.ReadAll(body)
+		body.Close()
+		sess.Logger().Warn("Status code is not 2xx in s3 response", zap.String("body", string(b)))
 	}
 
 	w.WriteHeader(resp.StatusCode)

boltrouter/bolt_request.go

@@ -31,7 +31,7 @@ func (br *BoltRouter) NewBoltRequest(ctx context.Context, req *http.Request) (*B
 		return nil, fmt.Errorf("could not get aws credentials: %w", err)
 	}
 
-	failoverRequest, err := newFailoverAwsRequest(ctx, req.Clone(ctx), awsCred, sourceBucket)
+	failoverRequest, err := newFailoverAwsRequest(ctx, req, awsCred, sourceBucket)
 	if err != nil {
 		return nil, fmt.Errorf("failed to make failover request: %w", err)
 	}
@@ -128,25 +128,30 @@ func newFailoverAwsRequest(ctx context.Context, req *http.Request, awsCred aws.C
 
 	}
 
-	req.Header.Del("Authorization")
-	req.Header.Del("X-Amz-Security-Token")
+	clone := req.Clone(ctx)
 
-	req.URL.Host = host
-	req.Host = host
-	req.URL.Scheme = "https"
-	req.RequestURI = ""
+	clone.Header.Del("Authorization")
+	clone.Header.Del("X-Amz-Security-Token")
+
+	clone.URL.Host = host
+	clone.Host = host
+	clone.URL.Scheme = "https"
+	clone.RequestURI = ""
 	// This needs to be set to "" in order to fix unicode errors in RawPath
 	// This forces to use the well formated req.URL.Path value instead
-	req.URL.RawPath = ""
+	clone.URL.RawPath = ""
+
+	// req.Clone(ctx) does not clone Body, need to clone body manually
+	CopyReqBody(req, clone)
 
 	payloadHash := req.Header.Get("X-Amz-Content-Sha256")
 
 	awsSigner := v4.NewSigner()
-	if err := awsSigner.SignHTTP(ctx, awsCred, req, payloadHash, "s3", sourceBucket.Region, time.Now()); err != nil {
+	if err := awsSigner.SignHTTP(ctx, awsCred, clone, payloadHash, "s3", sourceBucket.Region, time.Now()); err != nil {
 		return nil, err
 	}
 
-	return req.Clone(ctx), nil
+	return clone, nil
 }
 
 // DoBoltRequest sends an HTTP Bolt request and returns an HTTP response, following policy (such as redirects, cookies, auth) as configured on the client.
@@ -159,7 +164,7 @@ func (br *BoltRouter) DoBoltRequest(logger *zap.Logger, boltReq *BoltRequest) (*
 	} else if !StatusCodeIs2xx(resp.StatusCode) && br.config.Failover {
 		b, _ := io.ReadAll(resp.Body)
 		resp.Body.Close()
-		logger.Warn("bolt request failed", zap.Int("status code", resp.StatusCode), zap.String("msg", string(b)))
+		logger.Warn("bolt request failed", zap.Int("statusCode", resp.StatusCode), zap.String("body", string(b)))
 		resp, err := http.DefaultClient.Do(boltReq.Aws)
 		return resp, true, err
 	}

boltrouter/bolt_request_test.go

@@ -3,8 +3,10 @@ package boltrouter
 import (
 	"context"
 	"net/http"
+	"strings"
 	"testing"
 
+	"github.com/Pallinder/go-randomdata"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/zap/zaptest"
 )
@@ -31,8 +33,9 @@ func TestBoltRequest(t *testing.T) {
 			require.NoError(t, err)
 			br.RefreshEndpoints(ctx)
 			boltVars := br.boltVars
+			body := strings.NewReader(randomdata.Paragraph())
 
-			req, err := http.NewRequest(tt.httpMethod, "test.projectn.co", nil)
+			req, err := http.NewRequest(tt.httpMethod, "test.projectn.co", body)
 			req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential=AKIA3Y7DLM2EYWSYCN5P/20230511/us-west-2/s3/aws4_request, SignedHeaders=accept-encoding;amz-sdk-invocation-id;amz-sdk-request;host;x-amz-content-sha256;x-amz-date, Signature=6447287d46d333a010e224191d64c31b9738cc37886aadb7753a0a579a30edc6")
 			require.NoError(t, err)
 

boltrouter/copy_body.go

@@ -0,0 +1,25 @@
+package boltrouter
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+)
+
+// copyReq copies the request body into a destination request.
+// this allows reading a request body multiple times without "closing" it
+func CopyReqBody(src *http.Request, dest *http.Request) {
+	var b bytes.Buffer
+	b.ReadFrom(src.Body)
+	src.Body = io.NopCloser(&b)
+	dest.Body = io.NopCloser(bytes.NewReader(b.Bytes()))
+}
+
+// copyResp copies the response body and returns a new response with the copied body.
+// this allows reading a response body multiple times without "closing" it.
+func CopyRespBody(resp *http.Response) io.ReadCloser {
+	var b bytes.Buffer
+	b.ReadFrom(resp.Body)
+	resp.Body = io.NopCloser(&b)
+	return io.NopCloser(bytes.NewReader(b.Bytes()))
+}

integration_tests/aws/list_objects_v2.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/project-n-oss/sidekick/integration_tests/aws/utils"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
@@ -20,6 +21,7 @@ func (s *AwsSuite) listObjectsV2() {
 	ctx := s.ctx
 	t := s.T()
 	awsBucket := aws.String(utils.Bucket)
+	expectedKeys := testDataKeys()
 
 	utils.AssertAwsClients(t, ctx, "ListObjectsV2",
 		&s3.ListObjectsV2Input{
@@ -31,6 +33,7 @@ func (s *AwsSuite) listObjectsV2() {
 			for i, obj := range resp.Contents {
 				keys[i] = *obj.Key
 			}
+			assert.ElementsMatch(t, expectedKeys, keys)
 			return reflect.ValueOf(keys)
 		},
 	)