Skip to content

Commit 02d2e5e

Browse files
rgmzrgmz
committed
fix(azuredevops): detector wasn't working
1 parent 1f33957 commit 02d2e5e

File tree

4 files changed

+177
-65
lines changed

4 files changed

+177
-65
lines changed

‎pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken.go

Lines changed: 116 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,17 @@
1-
package azuredevopspersonalaccesstoken
1+
package azure_devops
22

33
import (
44
"context"
5+
"encoding/json"
6+
"errors"
57
"fmt"
8+
"io"
69
"net/http"
710
"strings"
811

912
regexp "github.com/wasilibs/go-re2"
1013

14+
"github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple"
1115
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
1216
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
1317
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -21,75 +25,143 @@ type Scanner struct {
2125
// Ensure the Scanner satisfies the interface at compile time.
2226
var _ detectors.Detector = (*Scanner)(nil)
2327

24-
var (
25-
defaultClient = common.SaneHttpClient()
26-
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
27-
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure"}) + `\b([0-9a-z]{52})\b`)
28-
orgPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure"}) + `\b([0-9a-zA-Z][0-9a-zA-Z-]{5,48}[0-9a-zA-Z])\b`)
29-
)
28+
func (s Scanner) Type() detectorspb.DetectorType {
29+
return detectorspb.DetectorType_AzureDevopsPersonalAccessToken
30+
}
31+
32+
func (s Scanner) Description() string {
33+
return "Azure DevOps is a suite of development tools provided by Microsoft. Personal Access Tokens (PATs) are used to authenticate and authorize access to Azure DevOps services and resources."
34+
}
3035

3136
// Keywords are used for efficiently pre-filtering chunks.
3237
// Use identifiers in the secret preferably, or the provider name.
3338
func (s Scanner) Keywords() []string {
34-
return []string{"azure"}
39+
return []string{"dev.azure.com", "az devops"}
3540
}
3641

42+
var (
43+
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
44+
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure", "az", "token", "pat"}) + `\b([a-z0-9]{52}|[a-zA-Z0-9]{84})\b`)
45+
orgPat = regexp.MustCompile(`dev\.azure\.com/([0-9a-zA-Z][0-9a-zA-Z-]{5,48}[0-9a-zA-Z])\b`)
46+
47+
invalidOrgCache = simple.NewCache[struct{}]()
48+
)
49+
3750
// FromData will find and optionally verify AzureDevopsPersonalAccessToken secrets in a given set of bytes.
3851
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
3952
dataStr := string(data)
4053

41-
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
42-
orgMatches := orgPat.FindAllStringSubmatch(dataStr, -1)
43-
44-
for _, match := range matches {
45-
resMatch := strings.TrimSpace(match[1])
46-
for _, orgMatch := range orgMatches {
47-
resOrgMatch := strings.TrimSpace(orgMatch[1])
54+
// Deduplicate results.
55+
keyMatches := make(map[string]struct{})
56+
for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) {
57+
m := match[1]
58+
if detectors.StringShannonEntropy(m) < 3 {
59+
continue
60+
}
61+
keyMatches[m] = struct{}{}
62+
}
63+
orgMatches := make(map[string]struct{})
64+
for _, match := range orgPat.FindAllStringSubmatch(dataStr, -1) {
65+
m := match[1]
66+
if invalidOrgCache.Exists(m) {
67+
continue
68+
}
69+
orgMatches[m] = struct{}{}
70+
}
4871

49-
s1 := detectors.Result{
72+
for key := range keyMatches {
73+
for org := range orgMatches {
74+
r := detectors.Result{
5075
DetectorType: detectorspb.DetectorType_AzureDevopsPersonalAccessToken,
51-
Raw: []byte(resMatch),
52-
RawV2: []byte(resMatch + resOrgMatch),
76+
Raw: []byte(key),
77+
RawV2: []byte(fmt.Sprintf(`{"organization":"%s","token":"%s"}`, org, key)),
5378
}
5479

5580
if verify {
56-
client := s.client
57-
if client == nil {
58-
client = defaultClient
59-
}
60-
req, err := http.NewRequestWithContext(ctx, "GET", "https://dev.azure.com/"+resOrgMatch+"/_apis/projects", nil)
61-
if err != nil {
62-
continue
81+
if s.client == nil {
82+
s.client = common.SaneHttpClient()
6383
}
64-
req.SetBasicAuth("", resMatch)
65-
res, err := client.Do(req)
66-
if err == nil {
67-
defer res.Body.Close()
68-
hasVerifiedRes, _ := common.ResponseContainsSubstring(res.Body, "lastUpdateTime")
69-
if res.StatusCode >= 200 && res.StatusCode < 300 && hasVerifiedRes {
70-
s1.Verified = true
71-
} else if res.StatusCode == 401 {
72-
// The secret is determinately not verified (nothing to do)
73-
} else {
74-
err = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
75-
s1.SetVerificationError(err, resMatch)
84+
85+
isVerified, extraData, verificationErr := verifyMatch(ctx, s.client, org, key)
86+
r.Verified = isVerified
87+
r.ExtraData = extraData
88+
if verificationErr != nil {
89+
if errors.Is(verificationErr, errInvalidOrg) {
90+
delete(orgMatches, org)
91+
invalidOrgCache.Set(org, struct{}{})
92+
continue
7693
}
77-
} else {
78-
s1.SetVerificationError(err, resMatch)
94+
r.SetVerificationError(verificationErr)
7995
}
8096
}
8197

82-
results = append(results, s1)
98+
results = append(results, r)
8399
}
84100
}
85101

86102
return results, nil
87103
}
88104

89-
func (s Scanner) Type() detectorspb.DetectorType {
90-
return detectorspb.DetectorType_AzureDevopsPersonalAccessToken
105+
var errInvalidOrg = errors.New("invalid organization")
106+
107+
func verifyMatch(ctx context.Context, client *http.Client, org string, key string) (bool, map[string]string, error) {
108+
req, err := http.NewRequestWithContext(ctx, "GET", "https://dev.azure.com/"+org+"/_apis/projects", nil)
109+
if err != nil {
110+
return false, nil, err
111+
}
112+
113+
req.SetBasicAuth("", key)
114+
req.Header.Set("Accept", "application/json")
115+
req.Header.Set("Content-Type", "application/json")
116+
res, err := client.Do(req)
117+
if err != nil {
118+
return false, nil, err
119+
}
120+
defer func() {
121+
_, _ = io.Copy(io.Discard, res.Body)
122+
_ = res.Body.Close()
123+
}()
124+
125+
switch res.StatusCode {
126+
case http.StatusOK:
127+
// {"count":1,"value":[{"id":"...","name":"Test","url":"https://dev.azure.com/...","state":"wellFormed","revision":11,"visibility":"private","lastUpdateTime":"2024-12-16T02:23:58.86Z"}]}
128+
var projectsRes listProjectsResponse
129+
if json.NewDecoder(res.Body).Decode(&projectsRes) != nil {
130+
return false, nil, err
131+
}
132+
133+
// Condense a list of organizations + roles.
134+
var (
135+
extraData map[string]string
136+
projects = make([]string, 0, len(projectsRes.Value))
137+
)
138+
for _, p := range projectsRes.Value {
139+
projects = append(projects, p.Name)
140+
}
141+
if len(projects) > 0 {
142+
extraData = map[string]string{
143+
"projects": strings.Join(projects, ","),
144+
}
145+
}
146+
return true, extraData, nil
147+
case http.StatusUnauthorized:
148+
// The secret is determinately not verified (nothing to do)
149+
return false, nil, nil
150+
case http.StatusNotFound:
151+
// Org doesn't exist.
152+
return false, nil, errInvalidOrg
153+
default:
154+
body, _ := io.ReadAll(res.Body)
155+
return false, nil, fmt.Errorf("unexpected HTTP response: status=%d, body=%q", res.StatusCode, string(body))
156+
}
91157
}
92158

93-
func (s Scanner) Description() string {
94-
return "Azure DevOps is a suite of development tools provided by Microsoft. Personal Access Tokens (PATs) are used to authenticate and authorize access to Azure DevOps services and resources."
159+
type listProjectsResponse struct {
160+
Count int `json:"count"`
161+
Value []projectResponse `json:"value"`
162+
}
163+
164+
type projectResponse struct {
165+
Id string `json:"id"`
166+
Name string `json:"name"`
95167
}

‎pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_integration_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
//go:build detectors
22
// +build detectors
33

4-
package azuredevopspersonalaccesstoken
4+
package azure_devops
55

66
import (
77
"context"

‎pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_test.go

Lines changed: 59 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package azuredevopspersonalaccesstoken
1+
package azure_devops
22

33
import (
44
"context"
@@ -10,19 +10,6 @@ import (
1010
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
1111
)
1212

13-
var (
14-
validPattern = `
15-
azure:
16-
azure_key: uie5tff7m5h5lqnqjhaltetqli90a08p6dhv9rn59uo30jgzw8un
17-
azure_org_id: WOkQXnjSxCyioEJRa8R6J39cN4Xfyy8CWl1BZksHYsevxVBFzG
18-
`
19-
invalidPattern = `
20-
azure:
21-
azure_key: uie5tff7m5H5lqnqjhaltetqli90a08p6dhv9rn59uo30jgzw8un
22-
azure_org_id: LOKi
23-
`
24-
)
25-
2613
func TestAzureDevopsPersonalAccessToken_Pattern(t *testing.T) {
2714
d := Scanner{}
2815
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
@@ -32,14 +19,67 @@ func TestAzureDevopsPersonalAccessToken_Pattern(t *testing.T) {
3219
input string
3320
want []string
3421
}{
22+
// old
23+
{
24+
name: "valid - old token",
25+
input: `
26+
provider "azuredevops" {
27+
# Configuration options
28+
org_service_url = "https://dev.azure.com/housemd"
29+
personal_access_token = "qkfon5cdjdekin4qnkgfr2nf367h6yjnnqm5upwqepd3rekl4l5a"
30+
}`,
31+
want: []string{"qkfon5cdjdekin4qnkgfr2nf367h6yjnnqm5upwqepd3rekl4l5a:housemd"},
32+
},
33+
34+
// new
3535
{
36-
name: "valid pattern",
37-
input: validPattern,
38-
want: []string{"uie5tff7m5h5lqnqjhaltetqli90a08p6dhv9rn59uo30jgzw8unWOkQXnjSxCyioEJRa8R6J39cN4Xfyy8CWl1BZksHYsevxVBFzG"},
36+
name: "valid - az devops CLI",
37+
input: ` echo "Tests failed. Creating a bug in Azure DevOps..."
38+
az devops login --organization https://dev.azure.com/TechServicesCorp --token A0us9bS1c6qe5blb6CT4FGRR4JcmPDg7uadVFmw4D65bvtdPcdVdJQQJ99AKACAAAAAPnX9AAAASAhDO4GFB
39+
az boards work-item create --title "Automated Bug: Test Failure" --type $(bugType) --description "Tests failed. See results.log for details." --project "Test"`,
40+
want: []string{"A0us9bS1c6qe5blb6CT4FGRR4JcmPDg7uadVFmw4D65bvtdPcdVdJQQJ99AKACAAAAAPnX9AAAASAhDO4GFB:TechServicesCorp"},
3941
},
4042
{
41-
name: "invalid pattern",
42-
input: invalidPattern,
43+
name: "valid - environment variables",
44+
input: `# Base image: Azure CLI with a lightweight Ubuntu distribution-mcr.microsoft.com/azure-cli:2.52.0
45+
FROM ubuntu:20.04
46+
47+
# Set environment variables for Azure DevOps agent
48+
ENV AZP_URL=https://dev.azure.com/EBOrg21
49+
ENV AZP_TOKEN=2ZGS1XLyxTU2wXlrXy71ldl1tBKceXM9kl6mVAeQchvWIErzkwtBJQjJ99AKACAAAAAAAAAAAAASAZDO5BA2
50+
ENV AZP_POOL=TestParty
51+
`,
52+
want: []string{"2ZGS1XLyxTU2wXlrXy71ldl1tBKceXM9kl6mVAeQchvWIErzkwtBJQjJ99AKACAAAAAAAAAAAAASAZDO5BA2:EBOrg21"},
53+
},
54+
{
55+
name: "valid - jupyter notebook",
56+
input: ` "4 https://dev.azure.com/SSGL-SMT/10_BG_AU5... "
57+
]
58+
},
59+
"execution_count": 3,
60+
"metadata": {},
61+
"output_type": "execute_result"
62+
}
63+
],
64+
"source": [
65+
"df.head()"
66+
]
67+
},
68+
{
69+
"cell_type": "code",
70+
"execution_count": 4,
71+
"metadata": {},
72+
"outputs": [],
73+
"source": [
74+
"token = r\"49QzGd2ZOLTWdoMc0S3M0cZkVVsBMTua01tlMYOkTUnEwxebgYdheQQJ99AKACAAAAAHsyrdAAASAZDOULjm\""
75+
]`,
76+
want: []string{"49QzGd2ZOLTWdoMc0S3M0cZkVVsBMTua01tlMYOkTUnEwxebgYdheQQJ99AKACAAAAAHsyrdAAASAZDOULjm:SSGL-SMT"},
77+
},
78+
79+
// Invalid
80+
{
81+
name: "invalid",
82+
input: `ssh.dev.azure.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC7Hr1oTWqNqOlzGJOfGJ4NakVyIzf1rXYd4d7wo6jBlkLvCA4odBlL0mDUyZ0/QUfTTqeu+tm22gOsv+VrVTMk6vwRU75gY/y9ut5Mb3bR5BV58dKXyq9A9UeB5Cakehn5Zgm6x1mKoVyf+FFn26iYqXJRgzIZZcZ5V6hrE0Qg39kZm4az48o0AUbf6Sp4SLdvnuMa2sVNwHBboS7EJkm57XQPVU3/QpyNLHbWDdzwtrlS+ez30S3AdYhLKEOxAG8weOnyrtLJAUen9mTkol8oII1edf7mWWbWVf0nBmly21+nZcmCTISQBtdcyPaEno7fFQMDD26/s0lfKob4Kw8H`,
4383
want: nil,
4484
},
4585
}

‎pkg/engine/defaults/defaults.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -897,7 +897,7 @@ func buildDetectorList() []detectors.Detector {
897897
&azure_entra_serviceprincipal_v2.Scanner{},
898898
&azure_batch.Scanner{},
899899
&azurecontainerregistry.Scanner{},
900-
&azuredevopspersonalaccesstoken.Scanner{},
900+
&azure_devops.Scanner{},
901901
// &azurefunctionkey.Scanner{}, // detector is throwing some FPs
902902
&azure_openai.Scanner{},
903903
&azuresearchadminkey.Scanner{},

0 commit comments

Comments
 (0)