[Fix] Upgraded AWS go-sdk to V2 (only for AWS Canary Token Verification)

go.mod

@@ -25,6 +25,10 @@ require (
 	github.com/alecthomas/kingpin/v2 v2.4.0
 	github.com/avast/apkparser v0.0.0-20240729092610-90591e0804ae
 	github.com/aws/aws-sdk-go v1.55.6
+	github.com/aws/aws-sdk-go-v2 v1.36.1
+	github.com/aws/aws-sdk-go-v2/config v1.29.6
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.59
+	github.com/aws/aws-sdk-go-v2/service/sns v1.33.19
 	github.com/aymanbagabas/go-osc52 v1.2.1
 	github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
 	github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
@@ -148,7 +152,16 @@ require (
 	github.com/andybalholm/brotli v1.1.1 // indirect
 	github.com/apache/arrow/go/v14 v14.0.2 // indirect
 	github.com/atotto/clipboard v0.1.4 // indirect
-	github.com/aws/smithy-go v1.20.1 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.24.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.33.14 // indirect
+	github.com/aws/smithy-go v1.22.2 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect

go.sum

@@ -118,8 +118,34 @@ github.com/avast/apkparser v0.0.0-20240729092610-90591e0804ae h1:rDNramK9mnAbvUB
 github.com/avast/apkparser v0.0.0-20240729092610-90591e0804ae/go.mod h1:GNvprXNmXaDjpHmN3RFxz5QdK5VXTUvmQludCbjoBy4=
 github.com/aws/aws-sdk-go v1.55.6 h1:cSg4pvZ3m8dgYcgqB97MrcdjUmZ1BeMYKUxMMB89IPk=
 github.com/aws/aws-sdk-go v1.55.6/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
-github.com/aws/smithy-go v1.20.1 h1:4SZlSlMr36UEqC7XOyRVb27XMeZubNcBNN+9IgEPIQw=
-github.com/aws/smithy-go v1.20.1/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
+github.com/aws/aws-sdk-go-v2 v1.36.1 h1:iTDl5U6oAhkNPba0e1t1hrwAo02ZMqbrGq4k5JBWM5E=
+github.com/aws/aws-sdk-go-v2 v1.36.1/go.mod h1:5PMILGVKiW32oDzjj6RU52yrNrDPUHcbZQYr1sM7qmM=
+github.com/aws/aws-sdk-go-v2/config v1.29.6 h1:fqgqEKK5HaZVWLQoLiC9Q+xDlSp+1LYidp6ybGE2OGg=
+github.com/aws/aws-sdk-go-v2/config v1.29.6/go.mod h1:Ft+WLODzDQmCTHDvqAH1JfC2xxbZ0MxpZAcJqmE1LTQ=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.59 h1:9btwmrt//Q6JcSdgJOLI98sdr5p7tssS9yAsGe8aKP4=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.59/go.mod h1:NM8fM6ovI3zak23UISdWidyZuI1ghNe2xjzUZAyT+08=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28 h1:KwsodFKVQTlI5EyhRSugALzsV6mG/SGrdjlMXSZSdso=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28/go.mod h1:EY3APf9MzygVhKuPXAc5H+MkGb8k/DOSQjWS0LgkKqI=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32 h1:BjUcr3X3K0wZPGFg2bxOWW3VPN8rkE3/61zhP+IHviA=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32/go.mod h1:80+OGC/bgzzFFTUmcuwD0lb4YutwQeKLFpmt6hoWapU=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32 h1:m1GeXHVMJsRsUAqG6HjZWx9dj7F5TR+cF1bjyfYyBd4=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32/go.mod h1:IitoQxGfaKdVLNg0hD8/DXmAqNy0H4K2H2Sf91ti8sI=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 h1:Pg9URiobXy85kgFev3og2CuOZ8JZUBENF+dcgWBaYNk=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 h1:D4oz8/CzT9bAEYtVhSBmFj2dNOtaHOtMKc2vHBwYizA=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2/go.mod h1:Za3IHqTQ+yNcRHxu1OFucBh0ACZT4j4VQFF0BqpZcLY=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13 h1:SYVGSFQHlchIcy6e7x12bsrxClCXSP5et8cqVhL8cuw=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13/go.mod h1:kizuDaLX37bG5WZaoxGPQR/LNFXpxp0vsUnqfkWXfNE=
+github.com/aws/aws-sdk-go-v2/service/sns v1.33.19 h1:ghgWtf6FnkD6YqDUq65Zg5lzQ92xADHBoJdWUyChiFw=
+github.com/aws/aws-sdk-go-v2/service/sns v1.33.19/go.mod h1:/TQAkYgLlLoH1/2Y9qgaE460iPWhdq67emlW/ue42U8=
+github.com/aws/aws-sdk-go-v2/service/sso v1.24.15 h1:/eE3DogBjYlvlbhd2ssWyeuovWunHLxfgw3s/OJa4GQ=
+github.com/aws/aws-sdk-go-v2/service/sso v1.24.15/go.mod h1:2PCJYpi7EKeA5SkStAmZlF6fi0uUABuhtF8ILHjGc3Y=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14 h1:M/zwXiL2iXUrHputuXgmO94TVNmcenPHxgLXLutodKE=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14/go.mod h1:RVwIw3y/IqxC2YEXSIkAzRDdEU1iRabDPaYjpGCbCGQ=
+github.com/aws/aws-sdk-go-v2/service/sts v1.33.14 h1:TzeR06UCMUq+KA3bDkujxK1GVGy+G8qQN/QVYzGLkQE=
+github.com/aws/aws-sdk-go-v2/service/sts v1.33.14/go.mod h1:dspXf/oYWGWo6DEvj98wpaTeqt5+DMidZD0A9BYTizc=
+github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ=
+github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
 github.com/aymanbagabas/go-osc52 v1.2.1 h1:q2sWUyDcozPLcLabEMd+a+7Ea2DitxZVN9hTxab9L4E=
 github.com/aymanbagabas/go-osc52 v1.2.1/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=

pkg/common/http.go

@@ -89,15 +89,15 @@ type CustomTransport struct {
 	T http.RoundTripper
 }
 
-func userAgent() string {
+func UserAgent() string {
 	if len(feature.UserAgentSuffix.Load()) > 0 {
 		return "TruffleHog " + feature.UserAgentSuffix.Load()
 	}
 	return "TruffleHog"
 }
 
 func (t *CustomTransport) RoundTrip(req *http.Request) (*http.Response, error) {
-	req.Header.Add("User-Agent", userAgent())
+	req.Header.Add("User-Agent", UserAgent())
 	return t.T.RoundTrip(req)
 }
 

pkg/detectors/aws/access_keys/accesskey.go

@@ -2,14 +2,17 @@ package access_keys
 
 import (
 	"context"
-	"encoding/hex"
-	"encoding/json"
 	"fmt"
-	"io"
+	"net"
 	"net/http"
 	"strings"
 	"time"
 
+	"github.com/aws/aws-sdk-go-v2/aws/middleware"
+	awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/sts"
 	regexp "github.com/wasilibs/go-re2"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -20,7 +23,7 @@ import (
 )
 
 type scanner struct {
-	verificationClient *http.Client
+	verificationClient config.HTTPClient
 	skipIDs            map[string]struct{}
 	detectors.DefaultMultiPartCredentialProvider
 }
@@ -55,7 +58,6 @@ var _ interface {
 } = (*scanner)(nil)
 
 var (
-	defaultVerificationClient = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	// Key types are from this list https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids
@@ -72,6 +74,32 @@ func (s scanner) Keywords() []string {
 	}
 }
 
+// The recommended way by AWS is to use the SDK's http client.
+// https://docs.aws.amazon.com/sdk-for-go/v2/developer-guide/configure-http.html
+// Note: Using default http.Client causes SignatureInvalid error in response. therefore, based on http default client implementation, we are using the same configuration.
+func getDefaultBuildableClient() *awshttp.BuildableClient {
+	return awshttp.NewBuildableClient().
+		WithTimeout(common.DefaultResponseTimeout).
+		WithDialerOptions(func(dialer *net.Dialer) {
+			dialer.Timeout = 2 * time.Second
+			dialer.KeepAlive = 5 * time.Second
+		}).
+		WithTransportOptions(func(tr *http.Transport) {
+			tr.Proxy = http.ProxyFromEnvironment
+			tr.MaxIdleConns = 5
+			tr.IdleConnTimeout = 5 * time.Second
+			tr.TLSHandshakeTimeout = 3 * time.Second
+			tr.ExpectContinueTimeout = 1 * time.Second
+		})
+}
+
+func (s scanner) getAWSBuilableClient() config.HTTPClient {
+	if s.verificationClient == nil {
+		s.verificationClient = getDefaultBuildableClient()
+	}
+	return s.verificationClient
+}
+
 // FromData will find and optionally verify AWS secrets in a given set of bytes.
 func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	logger := logContext.AddLogger(ctx).Logger().WithName("aws")
@@ -127,7 +155,7 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					isCanary = true
 					s1.ExtraData["message"] = thinkstMessage
 					if verify {
-						verified, arn, err := s.verifyCanary(idMatch, secretMatch)
+						verified, arn, err := s.verifyCanary(ctx, idMatch, secretMatch)
 						s1.Verified = verified
 						if arn != "" {
 							s1.ExtraData["arn"] = arn
@@ -139,7 +167,7 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					isCanary = true
 					s1.ExtraData["message"] = thinkstKnockoffsMessage
 					if verify {
-						verified, arn, err := s.verifyCanary(idMatch, secretMatch)
+						verified, arn, err := s.verifyCanary(ctx, idMatch, secretMatch)
 						s1.Verified = verified
 						if arn != "" {
 							s1.ExtraData["arn"] = arn
@@ -154,7 +182,7 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 
 			if verify && !isCanary {
-				isVerified, extraData, verificationErr := s.verifyMatch(ctx, idMatch, secretMatch, true)
+				isVerified, extraData, verificationErr := s.verifyMatch(ctx, idMatch, secretMatch, len(secretMatches) > 1)
 				s1.Verified = isVerified
 
 				// Log if the calculated ID does not match the ID value from verification.
@@ -199,117 +227,53 @@ const (
 )
 
 func (s scanner) verifyMatch(ctx context.Context, resIDMatch, resSecretMatch string, retryOn403 bool) (bool, map[string]string, error) {
-	// REQUEST VALUES.
-	now := time.Now().UTC()
-	datestamp := now.Format("20060102")
-	amzDate := now.Format("20060102T150405Z0700")
-
-	req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)
+	// Prep AWS Creds for STS
+	cfg, err := config.LoadDefaultConfig(ctx,
+		config.WithRegion(region),
+		config.WithHTTPClient(s.getAWSBuilableClient()),
+		config.WithCredentialsProvider(
+			credentials.NewStaticCredentialsProvider(resIDMatch, resSecretMatch, ""),
+		),
+	)
 	if err != nil {
 		return false, nil, err
 	}
-	req.Header.Set("Accept", "application/json")
-
-	// TASK 1: CREATE A CANONICAL REQUEST.
-	// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
-	canonicalURI := "/"
-	canonicalHeaders := "host:" + host + "\n"
-	signedHeaders := "host"
-	algorithm := "AWS4-HMAC-SHA256"
-	credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", datestamp, region, service)
-
-	params := req.URL.Query()
-	params.Add("Action", "GetCallerIdentity")
-	params.Add("Version", "2011-06-15")
-	params.Add("X-Amz-Algorithm", algorithm)
-	params.Add("X-Amz-Credential", resIDMatch+"/"+credentialScope)
-	params.Add("X-Amz-Date", amzDate)
-	params.Add("X-Amz-Expires", "30")
-	params.Add("X-Amz-SignedHeaders", signedHeaders)
-
-	canonicalQuerystring := params.Encode()
-	payloadHash := aws.GetHash("") // empty payload
-	canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQuerystring + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash
-
-	// TASK 2: CREATE THE STRING TO SIGN.
-	stringToSign := algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + aws.GetHash(canonicalRequest)
-
-	// TASK 3: CALCULATE THE SIGNATURE.
-	// https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
-	hash := aws.GetHMAC([]byte(fmt.Sprintf("AWS4%s", resSecretMatch)), []byte(datestamp))
-	hash = aws.GetHMAC(hash, []byte(region))
-	hash = aws.GetHMAC(hash, []byte(service))
-	hash = aws.GetHMAC(hash, []byte("aws4_request"))
-
-	signature2 := aws.GetHMAC(hash, []byte(stringToSign)) // Get Signature HMAC SHA256
-	signature := hex.EncodeToString(signature2)
-
-	// TASK 4: ADD SIGNING INFORMATION TO THE REQUEST.
-	params.Add("X-Amz-Signature", signature)
-	req.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8")
-	req.URL.RawQuery = params.Encode()
-
-	client := s.verificationClient
-	if client == nil {
-		client = defaultVerificationClient
-	}
+	// Create STS client
+	stsClient := sts.NewFromConfig(cfg, func(o *sts.Options) {
+		o.APIOptions = append(o.APIOptions, middleware.AddUserAgentKeyValue("User-Agent", common.UserAgent()))
+	})
 
-	res, err := client.Do(req)
+	// Make the GetCallerIdentity API call
+	resp, err := stsClient.GetCallerIdentity(ctx, &sts.GetCallerIdentityInput{})
 	if err != nil {
-		return false, nil, err
-	}
-	defer func() {
-		_, _ = io.Copy(io.Discard, res.Body)
-		_ = res.Body.Close()
-	}()
-
-	// TODO: tighten range of acceptable status codes
-	if res.StatusCode >= 200 && res.StatusCode < 300 {
-		identityInfo := aws.IdentityResponse{}
-		if err := json.NewDecoder(res.Body).Decode(&identityInfo); err != nil {
-			return false, nil, err
-		}
-
-		extraData := map[string]string{
-			"rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",
-			"account":        identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
-			"user_id":        identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
-			"arn":            identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
-		}
-		return true, extraData, nil
-	} else if res.StatusCode == 403 {
-		// Experimentation has indicated that if you make two GetCallerIdentity requests within five seconds that
+		// Experimentation has indicated that if you make multiple GetCallerIdentity requests within five seconds that
 		// share a key ID but are signed with different secrets the second one will be rejected with a 403 that
 		// carries a SignatureDoesNotMatch code in its body. This happens even if the second ID-secret pair is
 		// valid. Since this is exactly our access pattern, we need to work around it.
 		//
 		// Fortunately, experimentation has also revealed a workaround: simply resubmit the second request. The
-		// response to the resubmission will be as expected. But there's a caveat: You can't have closed the body of
-		// the response to the original second request, or read to its end, or the resubmission will also yield a
-		// SignatureDoesNotMatch. For this reason, we have to re-request all 403s. We can't re-request only
-		// SignatureDoesNotMatch responses, because we can only tell whether a given 403 is a SignatureDoesNotMatch
-		// after decoding its response body, which requires reading the entire response body, which disables the
-		// workaround.
+		// response to the resubmission will be as expected.
 		//
 		// We are clearly deep in the guts of AWS implementation details here, so this all might change with no
 		// notice. If you're here because something in this detector broke, you have my condolences.
-		if retryOn403 {
-			return s.verifyMatch(ctx, resIDMatch, resSecretMatch, false)
-		}
-
-		var body aws.ErrorResponseBody
-		if err = json.NewDecoder(res.Body).Decode(&body); err != nil {
-			return false, nil, fmt.Errorf("couldn't parse the sts response body (%v)", err)
-		}
-		// All instances of the code I've seen in the wild are PascalCased but this check is
-		// case-insensitive out of an abundance of caution
-		if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
+		if strings.Contains(err.Error(), "StatusCode: 403") {
+			if retryOn403 {
+				return s.verifyMatch(ctx, resIDMatch, resSecretMatch, false)
+			}
+			return false, nil, nil
+		} else if strings.Contains(err.Error(), "InvalidClientTokenId") {
 			return false, nil, nil
 		}
-		return false, nil, fmt.Errorf("request returned status %d with an unexpected reason (%s: %s)", res.StatusCode, body.Error.Code, body.Error.Message)
-	} else {
-		return false, nil, fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
+		return false, nil, fmt.Errorf("request returned unexpected error: %w", err)
+	}
+
+	extraData := map[string]string{
+		"rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",
+		"account":        *resp.Account,
+		"user_id":        *resp.UserId,
+		"arn":            *resp.Arn,
 	}
+	return true, extraData, nil
 }
 
 func (s scanner) CleanResults(results []detectors.Result) []detectors.Result {