Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Azure: reduce false-positive secrets #3722

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Azure: reduce false-positive secrets #3722

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 34 additions & 15 deletions pkg/detectors/azure_entra/common.go
View file
Original file line number Diff line number Diff line change
@@ -4,6 +4,7 @@ import (
"fmt"
"io"
"net/http"
stdRegexp "regexp" // Faster for small inputs.
"strings"

regexp "github.com/wasilibs/go-re2"
@@ -24,7 +25,7 @@ var (
// https://learn.microsoft.com/en-us/microsoft-365/admin/setup/domains-faq?view=o365-worldwide#why-do-i-have-an--onmicrosoft-com--domain
tenantIdPat = regexp.MustCompile(fmt.Sprintf(
//language=regexp
`(?i)(?:(?:login\.microsoftonline\.com/|(?:login|sts)\.windows\.net/|(?:t[ae]n[ae]nt(?:[ ._-]?id)?|\btid)(?:.|\s){0,60}?)(%s)|https?://(%s)|X-AnchorMailbox(?:.|\s){0,60}?@(%s)|/(%s)/(?:oauth2/v2\.0|B2C_1\w+|common|discovery|federationmetadata|kerberos|login|openid/|reprocess|resume|saml2|token|uxlogout|v2\.0|wsfed))`,
`(?i)(?:(?:login\.microsoft(?:online)?\.com/|(?:login|sts)\.windows\.net/|(?:t[ae]n[ae]nt(?:[ ._-]?id)?|\btid)(?:.|\s){0,60}?)(%s)|https?://(%s)|X-AnchorMailbox(?:.|\s){0,60}?@(%s)|/(%s)/(?:oauth2/v2\.0|B2C_1\w+|common|discovery|federationmetadata|kerberos|login|openid/|reprocess|resume|saml2|token|uxlogout|v2\.0|wsfed))`,
uuidStr,
uuidStr,
uuidStr,
@@ -33,25 +34,19 @@ var (
tenantOnMicrosoftPat = regexp.MustCompile(`([\w-]+\.onmicrosoft\.com)`)

clientIdPat = regexp.MustCompile(fmt.Sprintf(
`(?i)(?:(?:app(?:lication)?|client)(?:[ ._-]?id)?|username| -u)(?:.|\s){0,45}?(%s)`, uuidStr))
`(?i)(?:(?:api|https?)://(%s)/|myapps\.microsoft\.com/signin/(?:[\w-]+/)?(%s)|(?:[\w:=]{0,10}?(?:app(?:lication)?|cl[ie][ei]nt)(?:[ ._-]?id)?|username| -u)(?:.|\s){0,45}?(%s))`, uuidStr, uuidStr, uuidStr))
)

// FindTenantIdMatches returns a list of potential tenant IDs in the provided |data|.
func FindTenantIdMatches(data string) map[string]struct{} {
uniqueMatches := make(map[string]struct{})

for _, match := range tenantIdPat.FindAllStringSubmatch(data, -1) {
var m string
if match[1] != "" {
m = strings.ToLower(match[1])
} else if match[2] != "" {
m = strings.ToLower(match[2])
} else if match[3] != "" {
m = strings.ToLower(match[3])
} else if match[4] != "" {
m = strings.ToLower(match[4])
}
if _, ok := detectors.UuidFalsePositives[detectors.FalsePositive(m)]; ok {
m := strings.ToLower(firstNonEmptyMatch(match))

if detectors.StringShannonEntropy(m) < 3 {
continue
} else if _, ok := detectors.UuidFalsePositives[detectors.FalsePositive(m)]; ok {
continue
} else if detectors.StringShannonEntropy(m) < 3 {
continue
@@ -64,12 +59,22 @@ func FindTenantIdMatches(data string) map[string]struct{} {
return uniqueMatches
}

// language=regexp
const invalidClientPat = `(?i)(?:client[._-]?request[._-]?(?:id)?(?:.|\s){1,10}%s|cid-v1:%s)`

// FindClientIdMatches returns a list of potential client UUIDs in the provided |data|.
func FindClientIdMatches(data string) map[string]struct{} {
uniqueMatches := make(map[string]struct{})
for _, match := range clientIdPat.FindAllStringSubmatch(data, -1) {
m := strings.ToLower(match[1])
if _, ok := detectors.UuidFalsePositives[detectors.FalsePositive(m)]; ok {
m := strings.ToLower(firstNonEmptyMatch(match))

fpPat := stdRegexp.MustCompile(fmt.Sprintf(invalidClientPat, m, m))
if detectors.StringShannonEntropy(m) < 3 {
continue
} else if _, ok := detectors.UuidFalsePositives[detectors.FalsePositive(m)]; ok {
continue
} else if fpPat.MatchString(match[0]) {
// Ignore request context UUID. (https://stackoverflow.com/q/59425520)
continue
} else if detectors.StringShannonEntropy(m) < 3 {
continue
@@ -132,3 +137,17 @@ func queryTenant(ctx context.Context, client *http.Client, tenant string) bool {
return false
}
}

// firstNonEmptyMatch returns the index and value of the first non-empty match.
func firstNonEmptyMatch(matches []string) string {
if len(matches) <= 1 {
return ""
}
// The first index is the entire matched string.
for _, val := range matches[1:] {
if val != "" {
return val
}
}
return ""
}
72 changes: 72 additions & 0 deletions pkg/detectors/azure_entra/common_test.go
View file
Original file line number Diff line number Diff line change
@@ -12,6 +12,7 @@ type testCase struct {
}

func runPatTest(t *testing.T, tests map[string]testCase, matchFunc func(data string) map[string]struct{}) {
t.Parallel()
t.Helper()
for name, test := range tests {
t.Run(name, func(t *testing.T) {
@@ -99,6 +100,7 @@ tenant_id = "57aabdfc-6ce0-4828-94a2-9abe277892ec"`,
"974fde14-c3a4-481b-9b03-cfce18213a07": {},
},
},
// https://github.com/AzureAD/microsoft-authentication-library-for-dotnet/blob/b444dd5f8800c298016ddd9da2e6c05b0bf4b02c/tests/Microsoft.Identity.Test.Common/TestConstants.cs#L241-L245
"login.microsoftonline.com": {
Input: ` auth: {
authority: 'https://login.microsoftonline.com/7bb339cb-e94c-4a85-884c-48ebd9bb28c3',
@@ -114,6 +116,12 @@ tenant_id = "57aabdfc-6ce0-4828-94a2-9abe277892ec"`,
"7bb339cb-e94c-4a85-884c-48ebd9bb28c3": {},
},
},
"login.microsoft.com": {
Input: `# SYSTEM_CONFIGURATION_ISSUER_URI=https://login.microsoft.com/2b820e29-94a2-402f-b666-c88ebcc69eb4/v2.0`,
Expected: map[string]struct{}{
"2b820e29-94a2-402f-b666-c88ebcc69eb4": {},
},
},
"sts.windows.net": {
Input: `{
"aud": "00000003-0000-0000-c000-000000000000",
@@ -299,6 +307,14 @@ $ApplicationId = "1e002bca-c6e2-446e-a29e-a221909fe8aa"`,
"902aeb6d-29c7-4f6e-849d-4b933117e320": {},
},
},
"cleint (typo)": {
Input: ` console.log({
cleintId:
"f3a45cb9-e388-4358-a6ef-08a63f97457c",`,
Expected: map[string]struct{}{
"f3a45cb9-e388-4358-a6ef-08a63f97457c": {},
},
},
"clientid": {
Input: `export const msalConfig = {
auth: {
@@ -342,6 +358,47 @@ subscription_id = "47ab1364-000d-4a53-838d-1537b1e3b49f"
"21e144ac-532d-49ad-ba15-1c40694ce8b1": {},
},
},
"uri - api://": {
Input: `AUDIENCE=api://51aaa91a-bb09-40cd-9f1f-e8c0936246c6/.default`,
Expected: map[string]struct{}{
"51aaa91a-bb09-40cd-9f1f-e8c0936246c6": {},
},
},
"uri - http://": {
Input: `AUDIENCE=http://ceb233fb-f14c-4330-9c5b-91f7db4970e1/.default`,
Expected: map[string]struct{}{
"ceb233fb-f14c-4330-9c5b-91f7db4970e1": {},
},
},
"uri - https://": {
Input: `AUDIENCE=https://47c3cfeb-b7f4-466a-b690-f7fcc79472a9/.default`,
Expected: map[string]struct{}{
"47c3cfeb-b7f4-466a-b690-f7fcc79472a9": {},
},
},
"uri - myapps.microsoft.com": {
Input: `# Linkcheck configuration
linkcheck_ignore = [
"https://myapps.microsoft.com/signin/01501f0f-c48b-4327-92a2-2324949b0a9c?tenantId=e4cbd4d7-327c-47fc-bcde-1005207021a5",
]`,
Expected: map[string]struct{}{
"01501f0f-c48b-4327-92a2-2324949b0a9c": {},
},
},
"uri - myapps.microsoft.com with name": {
Input: `$LoginURL = 'https://myapps.microsoft.com/signin/App1/c370c8f6-0cb5-44b2-a903-c6cbd5ff6ce4?tenantId=74d7c41b-e3b6-4d40-88cf-436fd5fc231a'`,
Expected: map[string]struct{}{
"c370c8f6-0cb5-44b2-a903-c6cbd5ff6ce4": {},
},
},
// TODO
// "createdBy": {
// Input: ` "systemData": {
// "createdAt": "2023-08-21T00:30:04.2907836\u002B00:00",
// "createdBy": "117311a5-df69-4fff-a301-6be98c1bd0ab",
// "createdByType": "Application"
// }`,
// },

// Arbitrary test cases
"spacing": {
@@ -356,6 +413,21 @@ subscription_id = "47ab1364-000d-4a53-838d-1537b1e3b49f"
"f12345c6-7890-1f23-b456-789eb0bb1c23": {},
},
},

// Invalid
"invalid uri": {
Input: `# ldapadd -Y EXTERNAL -H ldapi:/// -f chrootpw.ldif`,
},
"invalid - AppInsights UUID": {
Input: ` "Date": "Mon, 21 Aug 2023 00:29:56 GMT",
"Request-Context": "appId=cid-v1:2d2e8e63-272e-4b3c-8598-4ee570a0e70d",
"Strict-Transport-Security": "max-age=15724800; includeSubDomains; preload",`,
},
"invalid - client-request-id": {
Input: ` "Accept-Encoding": "gzip, deflate",
"client-request-id": "c9e15037-e93c-4da9-b885-9641a826ed9a",
"Connection": "keep-alive",`,
},
}

runPatTest(t, cases, FindClientIdMatches)
2 changes: 1 addition & 1 deletion pkg/detectors/azure_entra/serviceprincipal/sp.go
View file
Original file line number Diff line number Diff line change
@@ -89,7 +89,7 @@ func VerifyCredentials(ctx context.Context, client *http.Client, tenantId string
if err := json.NewDecoder(res.Body).Decode(&errResp); err != nil {
return false, nil, err
}

switch res.StatusCode {
case http.StatusBadRequest, http.StatusUnauthorized:
// Error codes can be looked up by removing the `AADSTS` prefix.
45 changes: 27 additions & 18 deletions pkg/detectors/azure_entra/serviceprincipal/v1/spv1.go
View file
Original file line number Diff line number Diff line change
@@ -25,15 +25,13 @@ var _ interface {
detectors.Versioner
} = (*Scanner)(nil)

var (
defaultClient = common.SaneHttpClient()
// TODO: Azure storage access keys and investigate other types of creds.
// https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow#second-case-access-token-request-with-a-certificate
// https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow#third-case-access-token-request-with-a-federated-credential
//clientSecretPat = regexp.MustCompile(`(?i)(?:secret|password| -p[ =]).{0,80}?([\w~@[\]:.?*/+=-]{31,34}`)
// TODO: Tighten this regex and replace it with above.
secretPat = regexp.MustCompile(`(?i)(?:secret|password| -p[ =]).{0,80}[^A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]([A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]{31,34})[^A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]`)
)
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Azure
}

func (s Scanner) Description() string {
return serviceprincipal.Description
}

func (s Scanner) Version() int {
return 1
@@ -45,6 +43,19 @@ func (s Scanner) Keywords() []string {
return []string{"azure", "az", "entra", "msal", "login.microsoftonline.com", ".onmicrosoft.com"}
}

var (
defaultClient = common.SaneHttpClient()
// TODO: Azure storage access keys and investigate other types of creds.
// https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow#second-case-access-token-request-with-a-certificate
// https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow#third-case-access-token-request-with-a-federated-credential
// clientSecretPat = regexp.MustCompile(`(?i)(?:secret|password| -p[ =]).{0,80}?([\w~@[\]:.?*/+=-]{31,34}`)
// TODO: Tighten this regex and replace it with above.
secretPat = regexp.MustCompile(`(?i)(?:secret|password| -p[ =]).{0,80}[^A-Za-z0-9!#$%&()*+,\-./:;<>?@[\\\]^_{|}~]([A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]{31,34})(?:\s|\z|[^A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~])`)

invalidMatchPat = regexp.MustCompile(`^passwordCredentials":`)
invalidSecretPat = regexp.MustCompile(`^[a-zA-Z]+$`)
)

// FromData will find and optionally verify Azure secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -68,20 +79,18 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}

func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Azure
}

func (s Scanner) Description() string {
return serviceprincipal.Description
}

func findSecretMatches(data string) map[string]struct{} {
uniqueMatches := make(map[string]struct{})
for _, match := range secretPat.FindAllStringSubmatch(data, -1) {
m := match[1]
// Ignore secrets that are handled by the V2 detector.
if v2.SecretPat.MatchString(m) {
// Ignore secrets that are handled by the V2 detector.
continue
} else if detectors.StringShannonEntropy(m) < 3 {
// Ignore low-entropy results.
continue
} else if invalidSecretPat.MatchString(m) || invalidMatchPat.MatchString(match[0]) {
// Ignore patterns that are known to be false.
continue
}
uniqueMatches[m] = struct{}{}
62 changes: 49 additions & 13 deletions pkg/detectors/azure_entra/serviceprincipal/v1/spv1_test.go
View file
Original file line number Diff line number Diff line change
@@ -13,11 +13,24 @@ type testCase struct {

func Test_FindClientSecretMatches(t *testing.T) {
cases := map[string]testCase{
// secret
`secret`: {
Input: `"secret": "ljjK-62Q5bJbm43xU5At-NdeWDrhIO_28~",`,
Expected: map[string]struct{}{"ljjK-62Q5bJbm43xU5At-NdeWDrhIO_28~": {}},
},

// client secret
"client_secret": {
Input: ` "TenantId": "3d7e0652-b03d-4ed2-bf86-f1299cecde17",
"ClientSecret": "gHduiL_j6t4b6DG?Qr-G6M@IOS?mX3B9",`,
Expected: map[string]struct{}{"gHduiL_j6t4b6DG?Qr-G6M@IOS?mX3B9": {}},
},
"client secret at end": {
Input: `secret: UAByAGkAbQBhAHIAeQAgAEsAZQB5AA==`,
Expected: map[string]struct{}{
"UAByAGkAbQBhAHIAeQAgAEsAZQB5AA==": {},
},
},
"client_secret1": {
Input: ` public static string clientId = "413ff05b-6d54-41a7-9271-9f964bc10624";
public static string clientSecret = "k72~odcN_6TbVh5D~19_1Qkj~87trteArL";
@@ -72,10 +85,21 @@ configs = {"fs.azure.account.auth.type": "OAuth"`,
Input: ` "AZUREAD-AKS-APPID-SECRET": "8w__IGsaY.6g6jUxb1.pPGK262._pgX.q-",`,
Expected: map[string]struct{}{"8w__IGsaY.6g6jUxb1.pPGK262._pgX.q-": {}},
},
// "client_secret6": {
// Input: ``,
// Expected: map[string]struct{}{"": {}},
// },
"client_secret9": {
Input: ` client-id: 49abd816-45d1-479a-b49a-80bcf6d7213a
client-secret: 7.18gt1b2wO-t.~Cf.mlZCyHC7r_micnuO`,
Expected: map[string]struct{}{"7.18gt1b2wO-t.~Cf.mlZCyHC7r_micnuO": {}},
},
"client_secret10": {
Input: ` "aadClientSecret": "6p3t93TJzPgsNtQISqWc.-@?GCz9-ZWo",`,
Expected: map[string]struct{}{"6p3t93TJzPgsNtQISqWc.-@?GCz9-ZWo": {}},
},
"client_secret11": {
Input: `CLIENT_ID=9f5a8591-20b9-46b3-8317-9cc2cd52ca76
CLIENT_SECRET=2XlwmIlo3XkjE1@y[cuWsPj3_N@F23F/
TENANT_ID=08c43a10-b16a-426e-b7e8-b3b42a60e181`,
Expected: map[string]struct{}{"2XlwmIlo3XkjE1@y[cuWsPj3_N@F23F/": {}},
},

"password": {
Input: `# Login using Service Principal
@@ -86,18 +110,30 @@ $Credential = New-Object -TypeName System.Management.Automation.PSCredential -Ar
},

// False positives
"placeholder_secret": {
"invalid - placeholder_secret": {
Input: `- Log in with a service principal using a client secret:

az login --service-principal --username {{http://azure-cli-service-principal}} --password {{secret}} --tenant {{someone.onmicrosoft.com}}`,
Expected: nil,
},
// "client_secret3": {
// Input: ``,
// Expected: map[string]struct{}{
// "": {},
// },
// },
},

"invalid - only alpha characters": {
Input: `"passwordCredentials":[],"preferredTokenSigningKeyThumbprint":null,"publisherName":"Microsoft"`,
},
"invalid - low entropy": {
Input: `clientSecret: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`,
},
"invalid - passwordCredentials1": {
Input: `"passwordCredentials":[{"customKeyIdentifier":"UAByAGkAbQBhAHIAeQAgAEsAZQB5AA==","endDate":"2019-07-16T23:01:19.028Z","keyId":`,
},
"invalid - passwordCredentials2": {
Input: `"passwordCredentials":[{"customKeyIdentifier":"TQB5ACAARgBpAHIAcwB0ACAASwBlAHkA"`,
},
"invalid - passwordCredentials3": {
Input: `,"passwordCredentials":[{"customKeyIdentifier":"awBlAHkAZgBvAHIAaQBtAHAAYQBsAGEA",`,
},
"invalid - azure vault path": {
Input: ` public const string MsalArlingtonOBOKeyVaultUri = "https://msidlabs.vault.azure.net:443/secrets/ARLMSIDLAB1-IDLASBS-App-CC-Secret";`,
},
}

for name, test := range cases {
Loading
Oops, something went wrong.