Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Modified aws detector to use standard library #579

Merged
merged 1 commit into from
May 24, 2022
Merged

Modified aws detector to use standard library #579

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
153 changes: 103 additions & 50 deletions pkg/detectors/aws/aws.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,14 @@ package aws

import (
"context"
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"fmt"
"net/http"
"regexp"
"strings"

"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/sts"
"time"

"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
Expand All @@ -19,9 +22,12 @@ type Scanner struct{}
var _ detectors.Detector = (*Scanner)(nil)

var (
client = common.SaneHttpClient()

// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
// Key types are from this list https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids
keyPat = regexp.MustCompile(`\b((?:AKIA|ABIA|ACCA|ASIA)[0-9A-Z]{16})\b`)
secretPat = regexp.MustCompile(`\b([A-Za-z0-9+/]{40})\b`)
keyPat = regexp.MustCompile(`\b([A-Za-z0-9+/]{40})\b`)
idPat = regexp.MustCompile(`\b((?:AKIA|ABIA|ACCA|ASIA)[0-9A-Z]{16})\b`)
)

// Keywords are used for efficiently pre-filtering chunks.
Expand All @@ -35,67 +41,114 @@ func (s Scanner) Keywords() []string {
}
}

func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]detectors.Result, error) {
func GetHash(input string) string {
data := []byte(input)
hasher := sha256.New()
hasher.Write(data)
return (hex.EncodeToString(hasher.Sum(nil)))
}

func GetHMAC(key []byte, data []byte) []byte {
hasher := hmac.New(sha256.New, key)
hasher.Write(data)
return hasher.Sum(nil)
}

// FromData will find and optionally verify AWS secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
var results []detectors.Result

keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1)
secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
idMatches := idPat.FindAllStringSubmatch(dataStr, -1)

for _, keyMatch := range keyMatches {
if len(keyMatch) != 2 {
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])

key := strings.TrimSpace(keyMatch[1])
for _, idMatch := range idMatches {
if len(idMatch) != 2 {
continue
}

s := detectors.Result{
DetectorType: detectorspb.DetectorType_AWS,
Raw: []byte(key),
Redacted: key,
}
// TODO: Remove possible matches if they verify positive.
if verify {
for _, secretMatch := range secretMatches {
if len(secretMatch) != 2 {
continue
}
resIdMatch := strings.TrimSpace(idMatch[1])

secret := strings.TrimSpace(secretMatch[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_AWS,
Raw: []byte(resMatch),
}

result, err := callerIdentity(ctx, key, secret)
if verify {
//REQUEST VALUES
method := "GET"
service := "iam"
host := "iam.amazonaws.com"
region := "us-east-1"
endpoint := "https://iam.amazonaws.com"
datestamp := time.Now().UTC().Format("20060102")
amz_date := time.Now().UTC().Format("20060102T150405Z0700")

req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)
if err != nil {
// It also errors for signature mismatches on the client side before sending, and it's quite noisy.
continue
}
if result != nil && result.Account != nil {
s.Verified = true
break
}
}
}

if !s.Verified {
if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
continue
// TASK 1: CREATE A CANONICAL REQUEST
// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
canonical_uri := "/"
canonical_headers := "host:" + host + "\n"
signed_headers := "host"
algorithm := "AWS4-HMAC-SHA256"
credential_scope := fmt.Sprintf("%s/%s/%s/aws4_request", datestamp, region, service)

params := req.URL.Query()
params.Add("Action", "ListUsers")
params.Add("Version", "2010-05-08")
params.Add("X-Amz-Algorithm", algorithm)
params.Add("X-Amz-Credential", resIdMatch+"/"+credential_scope)
params.Add("X-Amz-Date", amz_date)
params.Add("X-Amz-Expires", "30")
params.Add("X-Amz-SignedHeaders", signed_headers)

canonical_querystring := params.Encode()
payload_hash := GetHash("") //empty payload
canonical_request := method + "\n" + canonical_uri + "\n" + canonical_querystring + "\n" + canonical_headers + "\n" + signed_headers + "\n" + payload_hash

// TASK 2: CREATE THE STRING TO SIGN
string_to_sign := algorithm + "\n" + amz_date + "\n" + credential_scope + "\n" + GetHash(canonical_request)

// TASK 3: CALCULATE THE SIGNATURE
//https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
hash := GetHMAC([]byte(fmt.Sprintf("AWS4%s", resMatch)), []byte(datestamp))
hash = GetHMAC(hash, []byte(region))
hash = GetHMAC(hash, []byte(service))
hash = GetHMAC(hash, []byte("aws4_request"))

signature2 := GetHMAC([]byte(hash), []byte(string_to_sign)) //Get Signature HMAC SHA256
signature := hex.EncodeToString(signature2)

// TASK 4: ADD SIGNING INFORMATION TO THE REQUEST
params.Add("X-Amz-Signature", signature)
req.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8")
req.URL.RawQuery = params.Encode()

res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears "random" enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
}

if len(secretMatches) > 0 {
results = append(results, s)
results = append(results, s1)
}
}

return detectors.CleanResults(results), nil
}

func callerIdentity(ctx context.Context, key, secret string) (*sts.GetCallerIdentityOutput, error) {
svc := sts.New(sts.Options{
HTTPClient: common.SaneHttpClient(),
Logger: nil,
Region: "us-west-2",
Credentials: credentials.NewStaticCredentialsProvider(key, secret, ""),
})
result, err := svc.GetCallerIdentity(ctx, nil)
return result, err
}
111 changes: 29 additions & 82 deletions pkg/detectors/aws/aws_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,97 +6,96 @@ import (
"testing"
"time"

"github.com/aws/aws-sdk-go-v2/service/sts"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"

"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)

func TestScanner_FromChunk(t *testing.T) {

func TestAWS_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("AWS_SECRET")
secretInactive := testSecrets.MustGetField("AWS_INACTIVE")
id := testSecrets.MustGetField("AWS")
secret := testSecrets.MustGetField("AWS")
id := testSecrets.MustGetField("AWS_ID")
inactiveSecret := testSecrets.MustGetField("AWS_INACTIVE")

type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
args args
wantSecrets []detectors.Result
wantErr bool
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "live key",
name: "found, verified",
s: Scanner{},
args: args{
ctx: ctx,
data: []byte(fmt.Sprintf("You can find a aws secret %s within awsId %s", secret, id)),

ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
verify: true,
},
wantSecrets: []detectors.Result{
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_AWS,
Verified: true,
Redacted: id,
},
},
wantErr: false,
},
{
name: "dead key",
name: "found, unverified",
s: Scanner{},
args: args{
ctx: ctx,
data: []byte(fmt.Sprintf("You can find a aws secret %s within awsId %s", secretInactive, id)),
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
verify: true,
},
wantSecrets: []detectors.Result{
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_AWS,
Verified: false,
Redacted: id,
},
},
wantErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: ctx,
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
wantSecrets: nil,
wantErr: false,
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Scanner.FromData() error = %v, wantErr %v", err, tt.wantErr)
t.Errorf("AWS.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatal("no raw secret present")
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.wantSecrets); diff != "" {
t.Errorf("%s: Scanner.FromData() diff: (-got +want)\n%s", tt.name, diff)
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("AWS.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
Expand All @@ -116,55 +115,3 @@ func BenchmarkFromData(benchmark *testing.B) {
})
}
}

func Test_callerIdentity(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("AWS_SECRET")
secretInactive := testSecrets.MustGetField("AWS_INACTIVE")
id := testSecrets.MustGetField("AWS")

type args struct {
key string
secret string
ctx context.Context
}
tests := []struct {
name string
args args
want *sts.GetCallerIdentityOutput
wantErr bool
}{
{
name: "invalid",
args: args{
key: id,
secret: secretInactive,
ctx: context.Background(),
},
wantErr: true,
},
{
name: "valid",
args: args{
key: id,
secret: secret,
ctx: context.Background(),
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := callerIdentity(tt.args.ctx, tt.args.key, tt.args.secret)
if (err != nil) != tt.wantErr {
t.Errorf("callerIdentity() error = %v, wantErr %v", err, tt.wantErr)
return
}
})
}
}