chore: change rulename to optional in custom rules and improve README

.2ms.yml

@@ -2815,3 +2815,59 @@ ignore-result:
 - e41df94f30916531be27748175d28f851f0bacad # test data from only_custom_rules.json
 - faf4f974fad32cc5fb928a6d94adfe141007e925 # test data from custom_rules_secrets.txt
 - 20734a351c9be94da9240871fa50cf0959cf94ba # test data from default_plus_non_override_rules.json
+- 05602cbb737e019ceaed1842a1fafa048f057e39 # test data from expectedReportWithIgnoredResults.json
+- 0911ff5aa4fb801ef72e9aec41274ff6642fb6cb # test data from defaultPlusNonOverrideRules.json
+- 09b533109d917e12c1474d6f6d7783561fb58d41 # test data from generic-api-keys.txt
+- 0a7fc5f23b68249d5c3aa5486c1a2caebeaac45e # test data from onlyCustomRules.json
+- 0bb0f0b115e826fc22e3058062fa5fb4a50f143b # test data from defaultPlusNonOverrideRules.json
+- 0d741fcc834c41f5650c3bd30abd750dca238aa9 # test data from onlyCustomRules.json
+- 10d99d86045759663429f5b5357877319fb62afb # test data from defaultPlusNonOverrideRules.json
+- 1150ca30a572ee85c486f042afbd078d677be261 # test data from defaultPlusNonOverrideRules.json
+- 1458d3fa01ddbd4681338b1f35dccb0b84160a02 # test data from onlyOverrideRules.json
+- 1577a912b253138103b9aad14d8a97c827750eaf # test data from expectedReportWithIgnoredResults.json
+- 16f450b8ed7bee3163796ccae2a86aff4516c740 # test data from defaultPlusAllCustomRules.json
+- 1cbf4baac00a479cf5fad6bdba794b5f4b33e984 # test data from defaultPlusNonOverrideRules.json
+- 1e29c09cf8f3a599a96f7d19659543a1587a0ee5 # test data from defaultPlusAllCustomRules.json
+- 241414fa731c9f86094d79292244c46f13e8f0e5 # test data from onlyDefaultIgnoreCustomRules.json
+- 2e021a1d6ea1b7c72177dae61918faf7f0ae902b # test data from generic-api-keys.txt
+- 310b6939ef5aa98f3a84130b777565470f5a03d6 # test data from onlyOverrideRules.json
+- 3881ff90e41473873bfbe2b9de1165c3bc642b3c # test data from expectedReportWithIgnoredResults.json
+- 429cd5eb6407c41b08eb4955f9b1775abe50bd5c # test data from defaultPlusAllCustomRules.json
+- 42d8be555f4cf15d452f3bd9b0b2e9549a012597 # test data from onlyOverrideRules.json
+- 4303f8965b423d7e652a58d480582cc4ba4a54a8 # test data from defaultPlusAllCustomRules.json
+- 4607cbe90fbc7552b0977298a96063ef3aec39c9 # test data from onlyCustomRules.json
+- 4ddac4f4b08377912a5de2323b6e376d63d80f3f # test data from expectedReportWithIgnoredResults.json
+- 5376ee01f078e3e9e2f94a174fef9a72e09caaaf # test data from defaultPlusAllCustomRules.json
+- 53fceac899c26c7ef5f008eb5693f95ffe07e7c4 # test data from expectedReportWithIgnoredResults.json
+- 5a4af3cfa60282d73f0864b2c8acc1403c832a33 # test data from onlyCustomRules.json
+- 5b7ff548f191f5437026d0e507f6b43cd037a320 # test data from defaultPlusAllCustomRules.json
+- 5ba90ad878a6770da26c654005e875aa4a373190 # test data from onlyOverrideRules.json
+- 5d1f1d3f613ae7862047c7c83170fa05fd244e43 # test data from defaultPlusAllCustomRules.json
+- 6ae0b6fa63c9827e787c4994107ee0cac421eb60 # test data from expectedReportWithIgnoredResults.json
+- 6b2b89be362c7d0ec4f06c615304b032f41250d0 # test data from expectedReportWithIgnoredResults.json
+- 6c0d8ef2eb9f751288ecf36a1024d612c1e27ab3 # test data from onlyCustomRules.json
+- 6c14ac093ccfdf9d878c79b4ea6376f0e4815a82 # test data from onlyCustomRules.json
+- 7345100bea8e537d8aa434d739a3c799ce4d2a62 # test data from onlyOverrideRules.json
+- 74eb842cf07b07fbe25093536654e7288b69c49a # test data from defaultPlusNonOverrideRules.json
+- 7903a5376d2c1179125c726e72dc96637dd49256 # test data from defaultPlusAllCustomRules.json
+- 8314bdcfa48aa5e09eca4ddc30c0d4715d5388dc # test data from expectedReportWithIgnoredResults.json
+- 86963a531def427b48e008d4333803a1ed857094 # test data from onlyOverrideRules.json
+- 8dfcd4aa7d4620bb02493b37ef82373dd940a662 # test data from onlyDefaultIgnoreCustomRules.json
+- 8f0baeb46dc7357b2ad677811cefe24bf27a45ed # test data from onlyOverrideRules.json
+- 9850a1398255e717ec5a76e08b861b0d3da87402 # test data from expectedReportWithIgnoredResults.json
+- 9afb2c9aece1398d6286625ef3a06e15b26147d4 # test data from defaultPlusNonOverrideRules.json
+- a415ed511dec492ee023608d76bc4e66934381de # test data from defaultPlusAllCustomRules.json
+- a49f52bcec28c23b51d9082c2001ed2ef401edc3 # test data from onlyCustomRules.json
+- b061d2c11bea74a82eb3fec7830843a2b3572d0d # test data from expectedReportWithIgnoredResults.json
+- bdcfaa5f2c91bf942042a507925e320dd43816ea # test data from onlyCustomRules.json
+- c028d45a6be26e44b7cf71ed25dd11ca0100800c # test data from expectedReportWithIgnoredResults.json
+- c66c28ce94415cd6f3efe40130f2ab08a7214087 # test data from onlyCustomNoOverrideRules.json
+- c6b5416e40ffd8bb1cf7d9d0c7965431144e00f8 # test data from defaultPlusAllCustomRules.json
+- ced63354eaca48df7cdf8ac571fdb6e25cefbaae # test data from onlyOverrideRules.json
+- d07771fdf69f59c61607f7994e30694f4ba25177 # test data from onlyCustomNoOverrideRules.json
+- d53aff50117f07d339749ebdc5556b117d82e5f5 # test data from defaultPlusNonOverrideRules.json
+- dac4cd38432590cee87d46e3448c874b7e2c70c5 # test data from defaultPlusAllCustomRules.json
+- dfbbddd73932c7210bfc953917c5b485e3ee7535 # test data from defaultPlusNonOverrideRules.json
+- f0544e7e9e25a6223cd10c37a445bfd4a4641337 # test data from defaultPlusAllCustomRules.json
+- f93dd5ab91efe7b70381afe3d4ebd1b26e628ac9 # test data from defaultPlusNonOverrideRules.json
+- ff81ccd6553feaf5fbbf573eac2a0042d05aaee4 # test data from defaultPlusNonOverrideRules.json

README.md

@@ -263,6 +263,71 @@ docker run -v $(pwd)/.2ms.yml:/app/.2ms.yml checkmarx/2ms \
 
 [![asciicast](https://asciinema.org/a/n8RHL4v6vI87uiUPZ9I7CgfYy.svg)](https://asciinema.org/a/n8RHL4v6vI87uiUPZ9I7CgfYy)
 
+## Custom Rules File
+
+We support custom rules, which are user defined rules that can be passed via a custom rules file using the `--custom-rules-path` flag. The custom rules file format and extension can be YAML or JSON.
+
+Custom rules can be:
+
+- **Overrides** - if a rule present in the file shares the same ruleId as a default rule of 2ms, the rule present in the file will replace (override) the default rule in the scan. 
+  - Note: If a rule is overridden, it will simply take all fields from the rule as defined in the file. You must include all fields that you want to be defined, otherwise they will be nil/empty.
+
+- **New rules** - if a rule does not share ruleId with a default rule, it will be appended to the list of rules used in the scan.
+
+Custom rules work properly with --rule and --ignore-rule flags. Rules can be selected/ignored by ruleId, ruleName and tag
+
+Regardless of being an override or new rule, a custom rule has the following required fields:
+- ruleId - unique identifier of the rule
+- ruleName - human readable name of the rule
+- regex - regex pattern used to identify the secret
+
+Other fields are optional and can be seen in the example bellow of a file with a custom rule
+
+**YAML Example:**
+```yaml
+- ruleId: 01ab7659-d25a-4a1c-9f98-dee9d0cf2e70 # REQUIRED: unique id, must match default rule id to override that default rule. Rule ids can be used as values in --rule and --ignore-rule flags
+  ruleName: Custom-Api-Key # should be human-readable name. If left empty for new rule, ruleName will take the value of ruleId. If left empty for override, default rule name will be considered. Rule names can be used as values in --rule and --ignore-rule flags 
+  description: Custom rule 
+  regex: (?i)\b\w*secret\w*\b\s*:?=\s*["']?([A-Za-z0-9/_+=-]{8,150})["']? # REQUIRED: golang regular expression used to find secrets. If capture group is present in regex, it used to find the secret, otherwise whole regex is used. which group is considered the secret can be defined with secretGroup
+  keywords: # Keywords are used for pre-regex check filtering. Rules that contain keywords will perform a quick string compare check to make sure the keyword(s) are in the content being scanned.
+    - access
+    - api
+  entropy: 3.5 # shannon entropy, measures how random a string is. The value will be higher the more random a string is. Default rules that use entropy have values between 2.0 and 4.5. Leave empty to consider matches regardless of entropy
+  secretGroup: 1 # defines which capture group of regex match is considered the secret. Is also used as the group that will have its entropy checked if `entropy` is set. Can be left empty, in which case the first capture group to match will be considered the secret
+  path: (?i)\.(?:tf|hcl)$ # regex to limit the rule to specific file paths. For example, only .tf and .hcl files
+  severity: High # severity, can only be one of [Critical, High, Medium, Low, Info]
+  tags: # identifiers for the rule, tags can be used as values of --rule and --ignore-rule flags
+    - api-key
+  scoreParameters: # scoreParameters can be omitted for overrides, in which case the respective default rule scoreParameters will be considered
+    category: General # category of the rule, should be a string of type ruledefine.RuleCategory. Impacts cvss score
+    ruleType: 4 # can go from 4 to 0, 4 being most severe. For overrides, if Category is defined, ruleType also needs to be defined, or otherwise it will be considered 0. Impacts cvss score
+  disableValidation: false # if true, disables validity check for this rule, regardless of --validate flag
+  deprecated: false # if true, the rule will not be used in the scan, regardless of --rule flag
+  allowLists: # allowed values to ignore if matched
+    - description: Allowlist for Custom Rule
+      matchCondition: OR # determines whether all criteria in the allowList must match. Can be AND or OR. Defaults to OR if not specified
+      regexTarget: match - # determines whether the regexes in allowList are tested against the rule.Regex match or the full line being scanned. Can be 'match' or 'line'. Defaults to 'match' if not specified
+      regexes: # allowed regex patterns
+        - (?i)(?:access(?:ibility|or)|access[_.-]?id|random[_.-]?access|api[_.-]?(?:id|name|version)|rapid|capital|[a-z0-9-]*?api[a-z0-9-]*?:jar:|author|X-MS-Exchange-Organization-Auth|Authentication-Results|(?:credentials?[_.-]?id|withCredentials)|(?:25[0-5]|2[0-4]\d|1?\d?\d)(?:\.(?:25[0-5]|2[0-4]\d|1?\d?\d)){3}|(?:bucket|foreign|hot|idx|natural|primary|pub(?:lic)?|schema|sequence)[_.-]?key|(?:turkey)|key[_.-]?(?:alias|board|code|frame|id|length|mesh|name|pair|press(?:ed)?|ring|selector|signature|size|stone|storetype|word|up|down|left|right)|KeyVault(?:[A-Za-z]*?(?:Administrator|Reader|Contributor|Owner|Operator|User|Officer))\s*[:=]\s*['"]?[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}['"]?|key[_.-]?vault[_.-]?(?:id|name)|keyVaultToStoreSecrets|key(?:store|tab)[_.-]?(?:file|path)|issuerkeyhash|(?-i:[DdMm]onkey|[DM]ONKEY)|keying|(?:secret)[_.-]?(?:length|name|size)|UserSecretsId|(?:csrf)[_.-]?token|(?:io\.jsonwebtoken[
+          \t]?:[
+          \t]?[\w-]+)|(?:api|credentials|token)[_.-]?(?:endpoint|ur[il])|public[_.-]?token|(?:key|token)[_.-]?file|(?-i:(?:[A-Z_]+=\n[A-Z_]+=|[a-z_]+=\n[a-z_]+=)(?:\n|\z))|(?-i:(?:[A-Z.]+=\n[A-Z.]+=|[a-z.]+=\n[a-z.]+=)(?:\n|\z)))
+      stopWords:  # stop words that if found in the secret, will discard the finding. Stop words are searched on the secret, which can be either the full regex match or the capture group if any is defined in the rule regex
+        - 000000,
+        - 6fe4476ee5a1832882e326b506d14126
+      paths: # paths that can be ignored for this allowList
+        - \.bb$
+        - \.bbappend$
+        - \.bbclass$
+        - \.inc$
+    - matchCondition: AND
+      regexTarget: line
+      regexes:
+        - LICENSE[^=]*=\s*"[^"]+
+        - LIC_FILES_CHKSUM[^=]*=\s*"[^"]+
+        - SRC[^=]*=\s*"[a-zA-Z0-9]+
+```
+
+
 ## Scan Commands
 
 The following sections describe the arguments used for scanning each of the supported platforms.

cmd/config_test.go

@@ -299,29 +299,25 @@ func TestCustomRulesFlag(t *testing.T) {
 			},
 			Tags: []string{"security", "credentials"},
 			ScoreParameters: ruledefine.ScoreParameters{
-				Category: "Secrets",
+				Category: "General",
 				RuleType: 2,
 			},
 			DisableValidation: true,
 			Deprecated:        true,
 		},
 		{
-			RuleID:      "b47a1995-6572-41bb-b01d-d215b43ab089",
-			RuleName:    "mock-rule2",
-			Description: "Match API keys",
-			Regex:       "[A-Za-z0-9]{40}",
-			Keywords:    []string{"api", "key"},
-			Entropy:     4.0,
-			Path:        "config/api_keys.yaml",
-			SecretGroup: 0,
-			Severity:    "Medium",
-			OldSeverity: "High",
-			AllowLists:  []*ruledefine.AllowList{},
-			Tags:        []string{"api", "custom"},
-			ScoreParameters: ruledefine.ScoreParameters{
-				Category: "API",
-				RuleType: 1,
-			},
+			RuleID:            "b47a1995-6572-41bb-b01d-d215b43ab089",
+			RuleName:          "mock-rule2",
+			Description:       "Match API keys",
+			Regex:             "[A-Za-z0-9]{40}",
+			Keywords:          []string{"api", "key"},
+			Entropy:           4.0,
+			Path:              "config/api_keys.yaml",
+			SecretGroup:       0,
+			Severity:          "Medium",
+			OldSeverity:       "High",
+			AllowLists:        []*ruledefine.AllowList{},
+			Tags:              []string{"api", "custom"},
 			DisableValidation: false,
 			Deprecated:        false,
 		},

cmd/main_test.go

@@ -58,7 +58,7 @@ func TestPreRun(t *testing.T) {
 			expectedPreRunErr: nil,
 		},
 		{
-			name: "errors on custom rules, rule name, id, regex missing",
+			name: "errors on custom rules, rule id and regex missing",
 			engineConfigVar: engine.EngineConfig{
 				CustomRules: []*ruledefine.Rule{
 					{
@@ -75,12 +75,11 @@ func TestPreRun(t *testing.T) {
 			expectedPreRunErr: nil,
 			expectedContainsInitErrs: []error{
 				fmt.Errorf("rule#0: missing ruleID"),
-				fmt.Errorf("rule#0: missing ruleName"),
 				fmt.Errorf("rule#0: missing regex"),
 			},
 		},
 		{
-			name: "errors on custom rules, regex and severity invalid",
+			name: "errors on custom rules, regex, severity and score parameters invalid",
 			engineConfigVar: engine.EngineConfig{
 				CustomRules: []*ruledefine.Rule{
 					{
@@ -89,6 +88,10 @@ func TestPreRun(t *testing.T) {
 						Description: "Match passwords",
 						Regex:       "[A-Za-z0-9]{32})",
 						Severity:    "mockSeverity",
+						ScoreParameters: ruledefine.ScoreParameters{
+							Category: "mockCategory",
+							RuleType: 10,
+						},
 					},
 					{
 						RuleID:      "b47a1995-6572-41bb-b01d-d215b43ab089",
@@ -103,6 +106,9 @@ func TestPreRun(t *testing.T) {
 				fmt.Errorf("rule#0;RuleID-db18ccf1-4fbf-49f6-aec1-939a2e5464c0: invalid regex"),
 				fmt.Errorf("rule#0;RuleID-db18ccf1-4fbf-49f6-aec1-939a2e5464c0: invalid severity:" +
 					" mockSeverity not one of ([Critical High Medium Low Info])"),
+				fmt.Errorf("rule#0;RuleID-db18ccf1-4fbf-49f6-aec1-939a2e5464c0: invalid category:" +
+					" mockCategory not an acceptable category of type RuleCategory"),
+				fmt.Errorf("rule#0;RuleID-db18ccf1-4fbf-49f6-aec1-939a2e5464c0: invalid rule type: 10 not an acceptable uint8 value, maximum is 4"),
 			},
 		},
 		{

cmd/testData/customRulesValid.json

@@ -22,7 +22,7 @@
     ],
     "tags": ["security", "credentials"],
     "scoreParameters": {
-      "category": "Secrets",
+      "category": "General",
       "ruleType": 2
     },
     "disableValidation": true,
@@ -41,10 +41,6 @@
     "oldSeverity": "High",
     "allowLists": [],
     "tags": ["api", "custom"],
-    "scoreParameters": {
-      "category": "API",
-      "ruleType": 1
-    },
     "disableValidation": false,
     "deprecated": false
   }

cmd/testData/customRulesValid.yaml

@@ -26,7 +26,7 @@
     - security
     - credentials
   scoreParameters:
-    category: Secrets
+    category: General
     ruleType: 2
   disableValidation: true
   deprecated: true
@@ -47,8 +47,5 @@
   tags:
     - api
     - custom
-  scoreParameters:
-    category: API
-    ruleType: 1
   disableValidation: false
   deprecated: false

engine/engine.go

@@ -6,6 +6,7 @@ import (
 	"crypto/hkdf"
 	"crypto/sha256"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -45,6 +46,12 @@ var (
 
 	ErrNoRulesSelected          = fmt.Errorf("no rules were selected")
 	ErrFailedToCompileRegexRule = fmt.Errorf("failed to compile regex rule")
+	errMissingRuleID            = fmt.Errorf("missing ruleID")
+	errMissingRegex             = fmt.Errorf("missing regex")
+	errInvalidRegex             = fmt.Errorf("invalid regex")
+	errInvalidSeverity          = fmt.Errorf("invalid severity")
+	errInvalidCategory          = fmt.Errorf("invalid category")
+	errInvalidRuleType          = fmt.Errorf("invalid rule type")
 )
 
 type DetectorConfig struct {
@@ -149,7 +156,7 @@ func Init(engineConfig *EngineConfig, opts ...EngineOption) (IEngine, error) {
 }
 
 func initEngine(engineConfig *EngineConfig, opts ...EngineOption) (*Engine, error) {
-	err := rules.CheckRulesRequiredFields(engineConfig.CustomRules)
+	err := CheckRulesRequiredFields(engineConfig.CustomRules)
 	if err != nil {
 		return nil, fmt.Errorf("failed to load custom rules: %w", err)
 	}
@@ -827,3 +834,63 @@ func isSecretFromConfluenceResourceIdentifier(secretRuleID, secretLine, secretMa
 	re := regexp.MustCompile(pat)
 	return re.MatchString(secretLine)
 }
+
+// CheckRulesRequiredFields checks that required fields are present in the Rule.
+// This is meant for user defined rules, default rules have more strict checks in unit tests
+func CheckRulesRequiredFields(rulesToCheck []*ruledefine.Rule) error {
+	var err error
+	for i, rule := range rulesToCheck {
+		if rule.RuleID == "" {
+			err = errors.Join(err, buildCustomRuleError(i, rule, errMissingRuleID))
+		}
+
+		if rule.Regex == "" {
+			err = errors.Join(err, buildCustomRuleError(i, rule, errMissingRegex))
+		} else {
+			if _, errRegex := regexp.Compile(rule.Regex); errRegex != nil {
+				invalidRegexError := fmt.Errorf("%w: %v", errInvalidRegex, errRegex)
+				err = errors.Join(err, buildCustomRuleError(i, rule, invalidRegexError))
+			}
+		}
+
+		if rule.Severity != "" {
+			if !slices.Contains(ruledefine.SeverityOrder, rule.Severity) {
+				invalidSeverityError := fmt.Errorf("%w: %s not one of (%s)", errInvalidSeverity, rule.Severity, ruledefine.SeverityOrder)
+				err = errors.Join(err, buildCustomRuleError(i, rule, invalidSeverityError))
+			}
+		}
+
+		if rule.ScoreParameters.Category != "" {
+			if _, ok := score.CategoryScoreMap[rule.ScoreParameters.Category]; !ok {
+				invalidCategoryError := fmt.Errorf("%w: %s not an acceptable category of type RuleCategory",
+					errInvalidCategory, rule.ScoreParameters.Category)
+				err = errors.Join(err, buildCustomRuleError(i, rule, invalidCategoryError))
+			}
+		}
+
+		if rule.ScoreParameters.RuleType != 0 {
+			if rule.ScoreParameters.RuleType > score.RuleTypeMaxValue {
+				invalidRuleTypeError := fmt.Errorf("%w: %d not an acceptable uint8 value, maximum is %d",
+					errInvalidRuleType, rule.ScoreParameters.RuleType, score.RuleTypeMaxValue)
+				err = errors.Join(err, buildCustomRuleError(i, rule, invalidRuleTypeError))
+			}
+		}
+	}
+
+	// Add a newline at start of error if it's not nil, for better presentation in output
+	if err != nil {
+		err = fmt.Errorf("\n%w", err)
+	}
+
+	return err
+}
+
+func buildCustomRuleError(ruleIndex int, rule *ruledefine.Rule, issue error) error {
+	if rule.RuleID == "" {
+		if rule.RuleName == "" {
+			return fmt.Errorf("rule#%d: %w", ruleIndex, issue)
+		}
+		return fmt.Errorf("rule#%d;RuleName-%s: %w", ruleIndex, rule.RuleName, issue)
+	}
+	return fmt.Errorf("rule#%d;RuleID-%s: %w", ruleIndex, rule.RuleID, issue)
+}