✨ [hashing] Add helpers to simplify hashing

.github/workflows/ci.yml

@@ -46,7 +46,7 @@ jobs:
     steps:
     - uses: actions/setup-python@v5
     - name: Install tools
-      run: pip install detect-secrets==1.0.3 && pip list
+      run: pip install detect-secrets[gibberish]==1.5.0 && pip list
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0

.secrets.baseline

@@ -1,5 +1,5 @@
 {
-  "version": "1.0.3",
+  "version": "1.5.0",
   "plugins_used": [
     {
       "name": "ArtifactoryDetector"
@@ -20,6 +20,15 @@
     {
       "name": "CloudantDetector"
     },
+    {
+      "name": "DiscordBotTokenDetector"
+    },
+    {
+      "name": "GitHubTokenDetector"
+    },
+    {
+      "name": "GitLabTokenDetector"
+    },
     {
       "name": "HexHighEntropyString",
       "limit": 3.0
@@ -30,6 +39,9 @@
     {
       "name": "IbmCosHmacDetector"
     },
+    {
+      "name": "IPPublicDetector"
+    },
     {
       "name": "JwtTokenDetector"
     },
@@ -43,9 +55,18 @@
     {
       "name": "NpmDetector"
     },
+    {
+      "name": "OpenAIDetector"
+    },
     {
       "name": "PrivateKeyDetector"
     },
+    {
+      "name": "PypiTokenDetector"
+    },
+    {
+      "name": "SendGridDetector"
+    },
     {
       "name": "SlackDetector"
     },
@@ -58,6 +79,9 @@
     {
       "name": "StripeDetector"
     },
+    {
+      "name": "TelegramBotTokenDetector"
+    },
     {
       "name": "TwilioKeyDetector"
     }
@@ -66,20 +90,26 @@
     {
       "path": "detect_secrets.filters.allowlist.is_line_allowlisted"
     },
-    {
-      "path": "detect_secrets.filters.common.is_baseline_file",
-      "filename": ".secrets.baseline"
-    },
     {
       "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
       "min_level": 2
     },
+    {
+      "path": "detect_secrets.filters.gibberish.should_exclude_secret",
+      "limit": 3.7
+    },
     {
       "path": "detect_secrets.filters.heuristic.is_indirect_reference"
     },
     {
       "path": "detect_secrets.filters.heuristic.is_likely_id_string"
     },
+    {
+      "path": "detect_secrets.filters.heuristic.is_lock_file"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string"
+    },
     {
       "path": "detect_secrets.filters.heuristic.is_potential_uuid"
     },
@@ -89,68 +119,25 @@
     {
       "path": "detect_secrets.filters.heuristic.is_sequential_string"
     },
+    {
+      "path": "detect_secrets.filters.heuristic.is_swagger_file"
+    },
     {
       "path": "detect_secrets.filters.heuristic.is_templated_secret"
     },
     {
       "path": "detect_secrets.filters.regex.should_exclude_file",
       "pattern": [
-        ".*go\\.sum$",
         ".*\\.html$",
         ".*\\.properties$",
-        "ci.yml",
-        "^workflows/.*",
-        "^\\.git[\\\\/]"
+        "^\\.git[\\\\/]",
+        ".*go\\.sum$",
+        ".*codegen.*",
+        "workflows/.*"
       ]
     }
   ],
   "results": {
-    "utils/config/fixtures/config-test.json": [
-      {
-        "type": "Secret Keyword",
-        "filename": "utils/config/fixtures/config-test.json",
-        "hashed_secret": "e38ad214943daad1d64c102faec29de4afe9da3d",
-        "is_verified": false,
-        "line_number": 10
-      },
-      {
-        "type": "Secret Keyword",
-        "filename": "utils/config/fixtures/config-test.json",
-        "hashed_secret": "2aa60a8ff7fcd473d321e0146afd9e26df395147",
-        "is_verified": false,
-        "line_number": 19
-      }
-    ],
-    "utils/config/service_configuration_test.go": [
-      {
-        "type": "Secret Keyword",
-        "filename": "utils/config/service_configuration_test.go",
-        "hashed_secret": "ddcec2f503a5d58f432a0beee3fb9544fa581f54",
-        "is_verified": false,
-        "line_number": 37
-      },
-      {
-        "type": "Secret Keyword",
-        "filename": "utils/config/service_configuration_test.go",
-        "hashed_secret": "7ca1cc114e7e5f955880bb96a5bf391b4dc20ab6",
-        "is_verified": false,
-        "line_number": 535
-      },
-      {
-        "type": "Secret Keyword",
-        "filename": "utils/config/service_configuration_test.go",
-        "hashed_secret": "11519c144be4850d95b34220a40030cbd5a36b57",
-        "is_verified": false,
-        "line_number": 630
-      },
-      {
-        "type": "Secret Keyword",
-        "filename": "utils/config/service_configuration_test.go",
-        "hashed_secret": "15fae91d8fa7f2c531c1cf3ddc745e1f4473c02d",
-        "is_verified": false,
-        "line_number": 637
-      }
-    ],
     "utils/filesystem/filehash_test.go": [
       {
         "type": "Hex High Entropy String",
@@ -215,7 +202,7 @@
         "filename": "utils/hashing/hash_test.go",
         "hashed_secret": "30f0cbefb37316806a7024caee994baf8365fa53",
         "is_verified": false,
-        "line_number": 132
+        "line_number": 136
       }
     ],
     "utils/sharedcache/common.go": [
@@ -276,5 +263,5 @@
       }
     ]
   },
-  "generated_at": "2025-09-26T12:33:39Z"
+  "generated_at": "2025-10-27T11:47:07Z"
 }

DEVELOPMENT.md

@@ -94,6 +94,6 @@ For the latter, False positive keys are stored in the [baseline](./.secrets.base
 
 To flag individual false positives add comment `# pragma: allowlist secret` to line with secret
 
-To add all suspected secrets in the repository (excluding ones with an allow secret comment), run `detect-secrets scan --all-files --exclude-files '.*go\.sum$' --exclude-files '.*\.html$' --exclude-files '.*\.properties$' --exclude-files 'ci.yml' --exclude-files '\.git' > .secrets.baseline`
+To add all suspected secrets in the repository (excluding ones with an allow secret comment), run `detect-secrets scan --all-files  --exclude-files '.*\.html$' --exclude-files '.*\.properties$'  --exclude-files '^\.git[\\/]' --exclude-files ".*go\.sum$" --exclude-files '.*codegen.*' --exclude-files 'workflows/.*'  > .secrets.baseline`
 
 If on Windows: then change the encoding of the .secrets.baseline file to UTF-8 then convert all `\` to `/` in the .secrets.baseline file

changes/20251027111917.feature

@@ -0,0 +1 @@
+:sparkles: [hashing] Add helpers to simplify hashing

utils/hashing/hash.go

@@ -6,6 +6,7 @@
 package hashing
 
 import (
+	"bytes"
 	"context"
 	"crypto/md5"  //nolint:gosec
 	"crypto/sha1" //nolint:gosec
@@ -36,6 +37,10 @@ const (
 	HashBlake2256 = "blake2b256" // https://www.blake2.net/
 )
 
+var (
+	SupportedHashingAlgorithms = []string{HashMd5, HashXXHash, HashSha1, HashSha256, HashMurmur, HashBlake2256}
+)
+
 type hashingAlgo struct {
 	Hash hash.Hash
 	Type string
@@ -133,7 +138,7 @@ func NewHashingAlgorithm(htype string) (IHash, error) {
 	}
 
 	if hash == nil {
-		return nil, commonerrors.New(commonerrors.ErrNotFound, "could not find the corresponding hashing algorithm")
+		return nil, commonerrors.Newf(commonerrors.ErrNotFound, "could not find the corresponding hashing algorithm. only %v are supported", SupportedHashingAlgorithms)
 	}
 	return newHashingAlgorithm(htype, hash)
 }
@@ -196,6 +201,21 @@ func CalculateHashWithContext(ctx context.Context, text, htype string) string {
 	return CalculateStringHashWithContext(ctx, hashing, text)
 }
 
+// CalculateHashFromReader returns the hash of element coming from a reader.
+func CalculateHashFromReader(ctx context.Context, htype string, reader io.Reader) (hash string, err error) {
+	hashing, err := NewHashingAlgorithm(htype)
+	if err != nil {
+		return
+	}
+	hash, err = hashing.CalculateWithContext(ctx, reader)
+	return
+}
+
+// CalculateBytesHash returns the hash of a byte array
+func CalculateBytesHash(ctx context.Context, htype string, array []byte) (string, error) {
+	return CalculateHashFromReader(ctx, htype, bytes.NewReader(array))
+}
+
 // CalculateHashOfListOfStrings calculates the hash of some text using the requested htype hashing algorithm.
 func CalculateHashOfListOfStrings(ctx context.Context, htype string, text ...string) string {
 	hashing, err := NewHashingAlgorithm(htype)

utils/hashing/hash_test.go

@@ -64,9 +64,13 @@ func TestMd5(t *testing.T) {
 	}, {
 		Input: "CMSIS",
 		Hash:  "c61d595888f85f6d30e99ef6cacfcb7d",
-	}}
+	},
+	}
 	for _, testCase := range testCases {
 		assert.Equal(t, testCase.Hash, CalculateMD5Hash(testCase.Input))
+		hash, err := CalculateBytesHash(context.Background(), HashMd5, []byte(testCase.Input))
+		require.NoError(t, err)
+		assert.Equal(t, testCase.Hash, hash)
 	}
 }