fix: generic support special character

.github/workflows/test.yml

@@ -1,9 +1,6 @@
 name: Test
 
 on:
-  push:
-    branches:
-      - "*"
   pull_request:
     branches:
       - "*"

.gitignore

@@ -21,3 +21,6 @@ testdata/expected/report/*.got.*
 *.out
 
 dist/
+
+#IDE
+.vscode/settings.json

cmd/generate/config/main.go

@@ -36,7 +36,8 @@ func main() {
 		rules.AsanaClientSecret(),
 		rules.Atlassian(),
 		rules.Authress(),
-		rules.AWS(),
+		rules.AWSAccessKey(),
+		rules.AWSSecretKey(),
 		rules.BitBucketClientID(),
 		rules.BitBucketClientSecret(),
 		rules.BittrexAccessKey(),
@@ -45,6 +46,9 @@ func main() {
 		rules.CodecovAccessToken(),
 		rules.CoinbaseAccessToken(),
 		rules.Clojars(),
+		rules.CloudflareAPIKey(),
+		rules.CloudflareGlobalAPIKey(),
+		rules.CloudflareOriginCAKey(),
 		rules.ConfluentAccessToken(),
 		rules.ConfluentSecretKey(),
 		rules.Contentful(),
@@ -143,6 +147,7 @@ func main() {
 		rules.Prefect(),
 		rules.PrivateKey(),
 		rules.PulumiAPIToken(),
+		rules.PuttyPrivateKey(),
 		rules.PyPiUploadToken(),
 		rules.RapidAPIAccessToken(),
 		rules.ReadMe(),

cmd/generate/config/rules/adobe.go

@@ -28,6 +28,7 @@ func AdobeClientSecret() *config.Rule {
 		RuleID:      "adobe-client-secret",
 		Regex:       generateUniqueTokenRegex(`(p8e-)(?i)[a-z0-9]{32}`, true),
 		Keywords:    []string{"p8e-"},
+		SecretGroup: 1,
 	}
 
 	// validate

cmd/generate/config/rules/alibaba.go

@@ -12,6 +12,7 @@ func AlibabaAccessKey() *config.Rule {
 		RuleID:      "alibaba-access-key-id",
 		Regex:       generateUniqueTokenRegex(`(LTAI)(?i)[a-z0-9]{20}`, true),
 		Keywords:    []string{"LTAI"},
+		SecretGroup: 1,
 	}
 
 	// validate

cmd/generate/config/rules/aws.go

@@ -1,18 +1,22 @@
 package rules
 
 import (
-	"regexp"
-
+	"github.com/zricethezav/gitleaks/v8/cmd/generate/secrets"
 	"github.com/zricethezav/gitleaks/v8/config"
 )
 
-func AWS() *config.Rule {
+// https://summitroute.com/blog/2018/06/20/aws_security_credential_formats/
+
+var credFileAccessKey = "aws_access_key_id=AKIALALEMEL33243OLIB" // gitleaks:allow
+var credFileSecretKey = "aws_secret_access_key=" + secrets.NewSecret(hex("40"))
+var credFileSessionToken = "aws_session_token=" + secrets.NewSecret(hex("928"))
+
+func AWSAccessKey() *config.Rule {
 	// define rule
 	r := config.Rule{
 		Description: "Identified a pattern that may indicate AWS credentials, risking unauthorized cloud resource access and data breaches on AWS platforms.",
-		RuleID:      "aws-access-token",
-		Regex: regexp.MustCompile(
-			"(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}"),
+		RuleID:      "aws-access-key",
+		Regex:       generateUniqueTokenRegex("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z2-7]{16}", false),
 		Keywords: []string{
 			"AKIA",
 			"ASIA",
@@ -22,6 +26,34 @@ func AWS() *config.Rule {
 	}
 
 	// validate
-	tps := []string{generateSampleSecret("AWS", "AKIALALEMEL33243OLIB")} // gitleaks:allow
-	return validate(r, tps, nil)
+	tps := []string{
+		generateSampleSecret("AWS", "AKIALALEMEL33243OLIB"), // gitleaks:allow
+		credFileAccessKey,
+	}
+	fps := []string{
+		generateSampleSecret("AWS", "AKIALALEMEL33243O000"), // includes 0 which can't be result of base32 encoding
+		`"RoleId": "AROAWORVRXQ5NC76T7223"`,
+		credFileSecretKey,
+		credFileSessionToken,
+	}
+	return validate(r, tps, fps)
+}
+
+func AWSSecretKey() *config.Rule {
+	// define rule
+	r := config.Rule{
+		Description: "Identified a pattern that may indicate AWS credentials, risking unauthorized cloud resource access and data breaches on AWS platforms.",
+		RuleID:      "aws-secret-key",
+		Regex:       generateUniqueTokenRegex("[0-9A-Z+\\/]{40}", true),
+	}
+
+	// validate
+	tps := []string{
+		credFileSecretKey,
+	}
+	fps := []string{
+		credFileAccessKey,
+		credFileSessionToken,
+	}
+	return validate(r, tps, fps)
 }

cmd/generate/config/rules/cloudflare.go

@@ -0,0 +1,76 @@
+package rules
+
+import (
+	"github.com/zricethezav/gitleaks/v8/config"
+)
+
+var global_keys = []string{
+	`cloudflare_global_api_key = "d3d1443e0adc9c24564c6c5676d679d47e2ca"`, // gitleaks:allow
+	`CLOUDFLARE_GLOBAL_API_KEY: 674538c7ecac77d064958a04a83d9e9db068c`,    // gitleaks:allow
+	`cloudflare: "0574b9f43978174cc2cb9a1068681225433c4"`,                 // gitleaks:allow
+}
+
+var api_keys = []string{
+	`cloudflare_api_key = "Bu0rrK-lerk6y0Suqo1qSqlDDajOk61wZchCkje4"`, // gitleaks:allow
+	`CLOUDFLARE_API_KEY: 5oK0U90ME14yU6CVxV90crvfqVlNH2wRKBwcLWDc`,    // gitleaks:allow
+	`cloudflare: "oj9Yoyq0zmOyWmPPob1aoY5YSNNuJ0fbZSOURBlX"`,          // gitleaks:allow
+}
+
+var origin_ca_keys = []string{
+	`CLOUDFLARE_ORIGIN_CA: v1.0-aaa334dc886f30631ba0a610-0d98ef66290d7e50aac7c27b5986c99e6f3f1084c881d8ac0eae5de1d1aa0644076ff57022069b3237d19afe60ad045f207ef2b16387ee37b749441b2ae2e9ebe5b4606e846475d4a5`,
+	`CLOUDFLARE_ORIGIN_CA: v1.0-15d20c7fccb4234ac5cdd756-d5c2630d1b606535cf9320ae7456b090e0896cec64169a92fae4e931ab0f72f111b2e4ffed5b2bb40f6fba6b2214df23b188a23693d59ce3fb0d28f7e89a2206d98271b002dac695ed`,
+}
+
+var identifiers = []string{"cloudflare"}
+
+func CloudflareGlobalAPIKey() *config.Rule {
+	// define rule
+	r := config.Rule{
+		Description: "Detected a Cloudflare Global API Key, potentially compromising cloud application deployments and operational security.",
+		RuleID:      "cloudflare-global-api-key",
+		Regex:       generateSemiGenericRegex(identifiers, hex("37"), true),
+
+		Keywords: identifiers,
+	}
+
+	// validate
+	tps := global_keys
+	fps := append(api_keys, origin_ca_keys...)
+
+	return validate(r, tps, fps)
+}
+
+func CloudflareAPIKey() *config.Rule {
+	// define rule
+	r := config.Rule{
+		Description: "Detected a Cloudflare API Key, potentially compromising cloud application deployments and operational security.",
+		RuleID:      "cloudflare-api-key",
+		Regex:       generateSemiGenericRegex(identifiers, alphaNumericExtendedShort("40"), true),
+
+		Keywords: identifiers,
+	}
+
+	// validate
+	tps := api_keys
+	fps := append(global_keys, origin_ca_keys...)
+
+	return validate(r, tps, fps)
+}
+
+func CloudflareOriginCAKey() *config.Rule {
+	ca_identifiers := append(identifiers, "v1.0-")
+	// define rule
+	r := config.Rule{
+		Description: "Detected a Cloudflare Origin CA Key, potentially compromising cloud application deployments and operational security.",
+		RuleID:      "cloudflare-origin-ca-key",
+		Regex:       generateUniqueTokenRegex(`v1\.0-`+hex("24")+"-"+hex("146"), false),
+
+		Keywords: ca_identifiers,
+	}
+
+	// validate
+	tps := origin_ca_keys
+	fps := append(global_keys, api_keys...)
+
+	return validate(r, tps, fps)
+}

cmd/generate/config/rules/generic.go

@@ -19,7 +19,7 @@ func GenericCredential() *config.Rule {
 			"password",
 			"auth",
 			"access",
-		}, `[0-9a-z\-_.=]{10,150}`, true),
+		}, `\S{10,150}`, true),
 		Keywords: []string{
 			"key",
 			"api",
@@ -43,12 +43,36 @@ func GenericCredential() *config.Rule {
 		generateSampleSecret("generic", "Zf3D0LXCM3EIMbgJpUNnkRtOfOueHznB"),
 		`"client_id" : "0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506"`,
 		`"client_secret" : "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde",`,
+		`"password: 'edf8f16608465858a6c9e3cccb97d3c2'"`,
+		// "<password>edf8f16608465858a6c9e3cccb97d3c2</password>",
+		`<element password="edf8f16608465858a6c9e3cccb97d3c2" />`,
+		"M_DB_PASSWORD= edf8f16608465858a6c9e3cccb97d3c2",
+		`{ "access-key": "6da89121079f83b2eb6acccf8219ea982c3d79bccc", }`,
+		`"{ \"access-key\": \"6da89121079f83b2eb6acccf8219ea982c3d79bccc\", }"`,
+		`access_key           = "kgfur834kmjfdoi34i9"`,
+		`TokenKey: b@d0@u7H50K3nx`,
+		`token_key: "gF[wSKyJmBhAFASD%3D"`,
+		`token = "weq32C232g37g2h3gdh3K2hT72hXuL2h3ghS34hD"`,
+		`client_secret = "F-oS9Su%}<>[];#"`,
 	}
 	fps := []string{
 		`client_vpn_endpoint_id = aws_ec2_client_vpn_endpoint.client-vpn-endpoint.id`,
 		`password combination.
 
-R5: Regulatory--21`,
+		R5: Regulatory--21`,
+
+		`"client_id" : "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"`,
+		// `"client_secret" : "4v7b9n2k5h",`, // entropy: 3.32
+		`"password: 'comp123!'"`,
+		"<password>MyComp9876</password>", // entropy: 3.32
+		`<element password="Comp4567@@" />`,
+		"M_DB_PASSWORD= aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+		"GITHUB_TOKEN: ${GITHUB_TOKEN}",
+		"password = 'your_password_here'",
+		"https://google.com?user=abc&password=123",
+		`"password": "abcdefg"`,                                   // short password
+		`api_key = "C71AAAAE-1D1D-1D1D-1D1D-1D1D1D1D1D1D"`,        // low entropy
+		`secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"`, // end with "Example Key" stop words
 	}
 	return validate(r, tps, fps)
 }

cmd/generate/config/rules/hashicorp.go

@@ -30,19 +30,28 @@ func HashicorpField() *config.Rule {
 	r := config.Rule{
 		Description: "Identified a HashiCorp Terraform password field, risking unauthorized infrastructure configuration and security breaches.",
 		RuleID:      "hashicorp-tf-password",
-		Regex:       generateSemiGenericRegex(keywords, fmt.Sprintf(`"%s"`, alphaNumericExtended("8,20")), true),
+		Regex:       generateSemiGenericRegex(keywords, fmt.Sprintf(`"%s"`, alphaNumericExtendedLong("8,20")), true),
 		Keywords:    keywords,
+		SecretGroup: 1,
+		Entropy:     3.5,
+		Allowlist: config.Allowlist{
+			StopWords: DefaultStopWords,
+		},
 	}
 
 	tps := []string{
 		// Example from: https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/sql_server.html
-		"administrator_login_password = " + `"thisIsDog11"`,
+		`administrator_login_password = "dgu6ju90k71r"`, // gitleaks:allow
 		// https://registry.terraform.io/providers/petoju/mysql/latest/docs
-		"password       = " + `"rootpasswd"`,
+		`password       = "gcerq4bcholjoh\s"`, // gitleaks:allow
 	}
 	fps := []string{
+		`administrator_login_password = "thisIsDog11"`, // entropy too low
+		`password       = "rootpasswd"`,                // entropy too low
 		"administrator_login_password = var.db_password",
 		`password = "${aws_db_instance.default.password}"`,
+		`update_password: "on_create"`,
+		// `const TagPassword = "dgu6ju90k71r"`, indeed it is a password, but it is not a terraform password field
 	}
 	return validate(r, tps, fps)
 }

cmd/generate/config/rules/heroku.go

@@ -17,6 +17,7 @@ func Heroku() *config.Rule {
 	// validate
 	tps := []string{
 		`const HEROKU_KEY = "12345678-ABCD-ABCD-ABCD-1234567890AB"`, // gitleaks:allow
+		`heroku_api_key = "832d2129-a846-4e27-99f4-7004b6ad53ef"`,   // gitleaks:allow
 	}
 	return validate(r, tps, nil)
 }

cmd/generate/config/rules/lob.go

@@ -17,6 +17,7 @@ func LobPubAPIToken() *config.Rule {
 			"live_pub",
 			"_pub",
 		},
+		SecretGroup: 1,
 	}
 
 	// validate
@@ -36,6 +37,7 @@ func LobAPIToken() *config.Rule {
 			"test_",
 			"live_",
 		},
+		SecretGroup: 1,
 	}
 
 	// validate