Adding a bunch of new rules, update allowlist to include node_modules… (

#896)

* Adding a bunch of new rules, update allowlist to include node_modules and vendor folders, extend helper config functions

* use func instead of function in stopwords

cmd/generate/config/main.go

@@ -29,40 +29,55 @@ func main() {
 	configRules = append(configRules, rules.BitBucketClientID())
 	configRules = append(configRules, rules.BitBucketClientSecret())
 	configRules = append(configRules, rules.Beamer())
+	configRules = append(configRules, rules.CoinbaseAccessToken())
 	configRules = append(configRules, rules.Clojars())
+	configRules = append(configRules, rules.ConfluentAccessToken())
+	configRules = append(configRules, rules.ConfluentSecretKey())
 	configRules = append(configRules, rules.Contentful())
 	configRules = append(configRules, rules.Databricks())
+	configRules = append(configRules, rules.DatadogtokenAccessToken())
 	configRules = append(configRules, rules.DiscordAPIToken())
 	configRules = append(configRules, rules.DiscordClientID())
 	configRules = append(configRules, rules.DiscordClientSecret())
+	configRules = append(configRules, rules.Doppler())
 	configRules = append(configRules, rules.DropBoxAPISecret())
 	configRules = append(configRules, rules.DropBoxLongLivedAPIToken())
 	configRules = append(configRules, rules.DropBoxShortLivedAPIToken())
-	configRules = append(configRules, rules.Doppler())
+	configRules = append(configRules, rules.DroneciAccessToken())
 	configRules = append(configRules, rules.Duffel())
 	configRules = append(configRules, rules.Dynatrace())
 	configRules = append(configRules, rules.EasyPost())
 	configRules = append(configRules, rules.EasyPostTestAPI())
+	configRules = append(configRules, rules.EtsyAccessToken())
 	configRules = append(configRules, rules.Facebook())
 	configRules = append(configRules, rules.FastlyAPIToken())
 	configRules = append(configRules, rules.FinicityClientSecret())
 	configRules = append(configRules, rules.FinicityAPIToken())
+	configRules = append(configRules, rules.FlickrAccessToken())
+	configRules = append(configRules, rules.FinnhubAccessToken())
 	configRules = append(configRules, rules.FlutterwavePublicKey())
 	configRules = append(configRules, rules.FlutterwaveSecretKey())
 	configRules = append(configRules, rules.FlutterwaveEncKey())
 	configRules = append(configRules, rules.FrameIO())
+	configRules = append(configRules, rules.FreshbooksAccessToken())
 	configRules = append(configRules, rules.GoCardless())
 	// TODO figure out what makes sense for GCP
 	// configRules = append(configRules, rules.GCPServiceAccount())
+	configRules = append(configRules, rules.GCPAPIKey())
 	configRules = append(configRules, rules.GitHubPat())
 	configRules = append(configRules, rules.GitHubOauth())
 	configRules = append(configRules, rules.GitHubApp())
 	configRules = append(configRules, rules.GitHubRefresh())
 	configRules = append(configRules, rules.Gitlab())
+	configRules = append(configRules, rules.GitterAccessToken())
 	configRules = append(configRules, rules.Hashicorp())
 	configRules = append(configRules, rules.Heroku())
 	configRules = append(configRules, rules.HubSpot())
 	configRules = append(configRules, rules.Intercom())
+	configRules = append(configRules, rules.KrakenAccessToken())
+	configRules = append(configRules, rules.KucoinAccessToken())
+	configRules = append(configRules, rules.KucoinSecretKey())
+	configRules = append(configRules, rules.LaunchDarklyAccessToken())
 	configRules = append(configRules, rules.LinearAPIToken())
 	configRules = append(configRules, rules.LinearClientSecret())
 	configRules = append(configRules, rules.LinkedinClientID())
@@ -74,12 +89,17 @@ func main() {
 	configRules = append(configRules, rules.MailGunPrivateAPIToken())
 	configRules = append(configRules, rules.MailGunSigningKey())
 	configRules = append(configRules, rules.MapBox())
+	configRules = append(configRules, rules.MattermostAccessToken())
 	configRules = append(configRules, rules.MessageBirdAPIToken())
 	configRules = append(configRules, rules.MessageBirdClientID())
+	configRules = append(configRules, rules.NetlifyAccessToken())
 	configRules = append(configRules, rules.NewRelicUserID())
 	configRules = append(configRules, rules.NewRelicUserKey())
 	configRules = append(configRules, rules.NewRelicBrowserAPIKey())
 	configRules = append(configRules, rules.NPM())
+	configRules = append(configRules, rules.NytimesAccessToken())
+	configRules = append(configRules, rules.PlaidAccessID())
+	configRules = append(configRules, rules.PlaidAccessToken())
 	configRules = append(configRules, rules.PlanetScalePassword())
 	configRules = append(configRules, rules.PlanetScaleAPIToken())
 	configRules = append(configRules, rules.PlanetScaleOAuthToken())
@@ -88,8 +108,12 @@ func main() {
 	configRules = append(configRules, rules.PulumiAPIToken())
 	configRules = append(configRules, rules.PyPiUploadToken())
 	configRules = append(configRules, rules.RubyGemsAPIToken())
+	configRules = append(configRules, rules.RapidAPIAccessToken())
+	configRules = append(configRules, rules.SendbirdAccessID())
+	configRules = append(configRules, rules.SendbirdAccessToken())
 	configRules = append(configRules, rules.SendGridAPIToken())
 	configRules = append(configRules, rules.SendInBlueAPIToken())
+	configRules = append(configRules, rules.SentryAccessToken())
 	configRules = append(configRules, rules.ShippoAPIToken())
 	configRules = append(configRules, rules.ShopifyAccessToken())
 	configRules = append(configRules, rules.ShopifyCustomAccessToken())
@@ -98,10 +122,21 @@ func main() {
 	configRules = append(configRules, rules.SlackAccessToken())
 	configRules = append(configRules, rules.SlackWebHook())
 	configRules = append(configRules, rules.StripeAccessToken())
+	configRules = append(configRules, rules.SquareAccessToken())
+	configRules = append(configRules, rules.SquareSpaceAccessToken())
+	configRules = append(configRules, rules.SumoLogicAccessID())
+	configRules = append(configRules, rules.SumoLogicAccessToken())
 	configRules = append(configRules, rules.Twilio())
 	configRules = append(configRules, rules.TwitchAPIToken())
-	configRules = append(configRules, rules.Twitter())
+	configRules = append(configRules, rules.TwitterAPIKey())
+	configRules = append(configRules, rules.TwitterAPISecret())
+	configRules = append(configRules, rules.TwitterAccessToken())
+	configRules = append(configRules, rules.TwitterAccessSecret())
+	configRules = append(configRules, rules.TwitterBearerToken())
 	configRules = append(configRules, rules.Typeform())
+	configRules = append(configRules, rules.YandexAPIKey())
+	configRules = append(configRules, rules.YandexAWSAccessToken())
+	configRules = append(configRules, rules.YandexAccessToken())
 	configRules = append(configRules, rules.GenericCredential())
 
 	// ensure rules have unique ids

cmd/generate/config/rules/coinbase.go

@@ -0,0 +1,27 @@
+package rules
+
+import (
+	"github.com/zricethezav/gitleaks/v8/cmd/generate/secrets"
+	"github.com/zricethezav/gitleaks/v8/config"
+)
+
+func CoinbaseAccessToken() *config.Rule {
+	// define rule
+	r := config.Rule{
+		RuleID:      "coinbase-access-token",
+		Description: "Coinbase Access Token",
+		Regex: generateSemiGenericRegex([]string{"coinbase"},
+			alphaNumericExtendedShort("64")),
+		SecretGroup: 1,
+		Keywords: []string{
+			"coinbase",
+		},
+	}
+
+	// validate
+	tps := []string{
+		generateSampleSecret("coinbase",
+			secrets.NewSecret(alphaNumericExtendedShort("64"))),
+	}
+	return validate(r, tps, nil)
+}

cmd/generate/config/rules/config.tmpl

@@ -19,8 +19,10 @@ regexes = [
     ]
 paths = [
     '''gitleaks.toml''',
-    '''(.*?)(jpg|gif|doc|pdf|bin|svg|socket)$''',
-    '''(go.mod|go.sum)$'''
+    '''(.*?)(jpg|gif|doc|docx|zip|xls|pdf|bin|svg|socket)$''',
+    '''(go.mod|go.sum)$''',
+    '''node_modules''',
+    '''vendor''',
 ]
 
 {{ range $i, $rule := .Rules }}[[rules]]

cmd/generate/config/rules/confluent.go

@@ -0,0 +1,44 @@
+package rules
+
+import (
+	"github.com/zricethezav/gitleaks/v8/cmd/generate/secrets"
+	"github.com/zricethezav/gitleaks/v8/config"
+)
+
+func ConfluentSecretKey() *config.Rule {
+	// define rule
+	r := config.Rule{
+		RuleID:      "confluent-secret-key",
+		Description: "Confluent Secret Key",
+		Regex:       generateSemiGenericRegex([]string{"confluent"}, alphaNumeric("64")),
+		SecretGroup: 1,
+		Keywords: []string{
+			"confluent",
+		},
+	}
+
+	// validate
+	tps := []string{
+		generateSampleSecret("confluent", secrets.NewSecret(alphaNumeric("64"))),
+	}
+	return validate(r, tps, nil)
+}
+
+func ConfluentAccessToken() *config.Rule {
+	// define rule
+	r := config.Rule{
+		RuleID:      "confluent-access-token",
+		Description: "Confluent Access Token",
+		Regex:       generateSemiGenericRegex([]string{"confluent"}, alphaNumeric("16")),
+		SecretGroup: 1,
+		Keywords: []string{
+			"confluent",
+		},
+	}
+
+	// validate
+	tps := []string{
+		generateSampleSecret("confluent", secrets.NewSecret(alphaNumeric("16"))),
+	}
+	return validate(r, tps, nil)
+}

cmd/generate/config/rules/datadog.go

@@ -0,0 +1,26 @@
+package rules
+
+import (
+	"github.com/zricethezav/gitleaks/v8/cmd/generate/secrets"
+	"github.com/zricethezav/gitleaks/v8/config"
+)
+
+func DatadogtokenAccessToken() *config.Rule {
+	// define rule
+	r := config.Rule{
+		RuleID:      "datadogtoken-access-token",
+		Description: "Datadogtoken Access Token",
+		Regex: generateSemiGenericRegex([]string{"datadog"},
+			alphaNumeric("40")),
+		SecretGroup: 1,
+		Keywords: []string{
+			"datadog",
+		},
+	}
+
+	// validate
+	tps := []string{
+		generateSampleSecret("datadog", secrets.NewSecret(alphaNumeric("40"))),
+	}
+	return validate(r, tps, nil)
+}

cmd/generate/config/rules/droneci.go

@@ -0,0 +1,25 @@
+package rules
+
+import (
+	"github.com/zricethezav/gitleaks/v8/cmd/generate/secrets"
+	"github.com/zricethezav/gitleaks/v8/config"
+)
+
+func DroneciAccessToken() *config.Rule {
+	// define rule
+	r := config.Rule{
+		RuleID:      "droneci-access-token",
+		Description: "Droneci Access Token",
+		Regex:       generateSemiGenericRegex([]string{"droneci"}, alphaNumeric("32")),
+		SecretGroup: 1,
+		Keywords: []string{
+			"droneci",
+		},
+	}
+
+	// validate
+	tps := []string{
+		generateSampleSecret("droneci", secrets.NewSecret(alphaNumeric("32"))),
+	}
+	return validate(r, tps, nil)
+}

cmd/generate/config/rules/etsy.go

@@ -0,0 +1,25 @@
+package rules
+
+import (
+	"github.com/zricethezav/gitleaks/v8/cmd/generate/secrets"
+	"github.com/zricethezav/gitleaks/v8/config"
+)
+
+func EtsyAccessToken() *config.Rule {
+	// define rule
+	r := config.Rule{
+		RuleID:      "etsy-access-token",
+		Description: "Etsy Access Token",
+		Regex:       generateSemiGenericRegex([]string{"etsy"}, alphaNumeric("24")),
+		SecretGroup: 1,
+		Keywords: []string{
+			"etsy",
+		},
+	}
+
+	// validate
+	tps := []string{
+		generateSampleSecret("etsy", secrets.NewSecret(alphaNumeric("24"))),
+	}
+	return validate(r, tps, nil)
+}

cmd/generate/config/rules/finnhub.go

@@ -0,0 +1,25 @@
+package rules
+
+import (
+	"github.com/zricethezav/gitleaks/v8/cmd/generate/secrets"
+	"github.com/zricethezav/gitleaks/v8/config"
+)
+
+func FinnhubAccessToken() *config.Rule {
+	// define rule
+	r := config.Rule{
+		RuleID:      "finnhub-access-token",
+		Description: "Finnhub Access Token",
+		Regex:       generateSemiGenericRegex([]string{"finnhub"}, alphaNumeric("20")),
+		SecretGroup: 1,
+		Keywords: []string{
+			"finnhub",
+		},
+	}
+
+	// validate
+	tps := []string{
+		generateSampleSecret("finnhub", secrets.NewSecret(alphaNumeric("20"))),
+	}
+	return validate(r, tps, nil)
+}

cmd/generate/config/rules/flickr.go

@@ -0,0 +1,25 @@
+package rules
+
+import (
+	"github.com/zricethezav/gitleaks/v8/cmd/generate/secrets"
+	"github.com/zricethezav/gitleaks/v8/config"
+)
+
+func FlickrAccessToken() *config.Rule {
+	// define rule
+	r := config.Rule{
+		RuleID:      "flickr-access-token",
+		Description: "Flickr Access Token",
+		Regex:       generateSemiGenericRegex([]string{"flickr"}, alphaNumeric("32")),
+		SecretGroup: 1,
+		Keywords: []string{
+			"flickr",
+		},
+	}
+
+	// validate
+	tps := []string{
+		generateSampleSecret("flickr", secrets.NewSecret(alphaNumeric("32"))),
+	}
+	return validate(r, tps, nil)
+}

cmd/generate/config/rules/freshbooks.go

@@ -0,0 +1,25 @@
+package rules
+
+import (
+	"github.com/zricethezav/gitleaks/v8/cmd/generate/secrets"
+	"github.com/zricethezav/gitleaks/v8/config"
+)
+
+func FreshbooksAccessToken() *config.Rule {
+	// define rule
+	r := config.Rule{
+		RuleID:      "freshbooks-access-token",
+		Description: "Freshbooks Access Token",
+		Regex:       generateSemiGenericRegex([]string{"freshbooks"}, alphaNumeric("64")),
+		SecretGroup: 1,
+		Keywords: []string{
+			"freshbooks",
+		},
+	}
+
+	// validate
+	tps := []string{
+		generateSampleSecret("freshbooks", secrets.NewSecret(alphaNumeric("64"))),
+	}
+	return validate(r, tps, nil)
+}

cmd/generate/config/rules/gcp.go

@@ -3,6 +3,7 @@ package rules
 import (
 	"regexp"
 
+	"github.com/zricethezav/gitleaks/v8/cmd/generate/secrets"
 	"github.com/zricethezav/gitleaks/v8/config"
 )
 
@@ -22,3 +23,22 @@ func GCPServiceAccount() *config.Rule {
 	}
 	return validate(r, tps, nil)
 }
+
+func GCPAPIKey() *config.Rule {
+	// define rule
+	r := config.Rule{
+		RuleID:      "gcp-api-key",
+		Description: "GCP API key",
+		Regex:       generateUniqueTokenRegex(`AIza[0-9A-Za-z\\-_]{35}`),
+		SecretGroup: 1,
+		Keywords: []string{
+			"AIza",
+		},
+	}
+
+	// validate
+	tps := []string{
+		generateSampleSecret("gcp", secrets.NewSecret(`AIza[0-9A-Za-z\\-_]{35}`)),
+	}
+	return validate(r, tps, nil)
+}