From 4b4f347e14188fc291d7faf1c47a67f114bb621d Mon Sep 17 00:00:00 2001 From: Pavlos Vinieratos Date: Thu, 3 Feb 2022 12:30:13 +0200 Subject: [PATCH] fix(detect-secrets): improve how secret detection works (#6109) * changes * more * some more * back to bash * rename * fix * Delete requirements.txt * Update scripts/install * Update .husky/pre-commit * Update package.json * small update --- .circleci/config.yml | 2 +- .husky/pre-commit | 2 +- .secrets.baseline | 311 ++++++++++++++++++++++- docs/troubleshooting.md | 8 +- package.json | 9 +- requirements.txt | 1 - scripts/detect-secrets-all | 5 - scripts/detect-secrets-generate-baseline | 5 - scripts/detect-secrets-staged | 5 - scripts/install | 2 +- scripts/secrets-add-all | 15 ++ scripts/secrets-add-staged | 15 ++ scripts/secrets-check-all | 10 + scripts/secrets-check-staged | 10 + scripts/secrets-generate-baseline | 13 + yarn.lock | 8 - 16 files changed, 378 insertions(+), 43 deletions(-) delete mode 100644 requirements.txt delete mode 100755 scripts/detect-secrets-all delete mode 100755 scripts/detect-secrets-generate-baseline delete mode 100755 scripts/detect-secrets-staged create mode 100755 scripts/secrets-add-all create mode 100755 scripts/secrets-add-staged create mode 100755 scripts/secrets-check-all create mode 100755 scripts/secrets-check-staged create mode 100755 scripts/secrets-generate-baseline diff --git a/.circleci/config.yml b/.circleci/config.yml index ad3d1ec72bd..0dee8e67cf1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -221,7 +221,7 @@ jobs: working_directory: /usr/src/app steps: - checkout - - run: detect-secrets-hook --baseline .secrets.baseline --exclude-secrets '[a-fA-F0-9]{24}' --exclude-lines 'W/"[!#-\x7E]*"' $(git ls-files | grep -v stickerpack) + - run: ./scripts/secrets-check-all deploy-nightly-beta: environment: diff --git a/.husky/pre-commit b/.husky/pre-commit index c5350077338..9a9bc286e80 100755 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -3,4 +3,4 @@ yarn lint-staged -yarn detect-secrets-staged +yarn secrets:check:staged diff --git a/.secrets.baseline b/.secrets.baseline index fcc645877dd..46e4aa40ce4 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -102,15 +102,14 @@ "path": "detect_secrets.filters.heuristic.is_templated_secret" }, { - "path": "detect_secrets.filters.regex.should_exclude_line", + "path": "detect_secrets.filters.regex.should_exclude_file", "pattern": [ - "W/[!#-\\x7E]*" - ] - }, - { - "path": "detect_secrets.filters.regex.should_exclude_secret", - "pattern": [ - "[a-fA-F0-9]{24}" + "/.lock$/", + "/.png$/", + "/.webp$/", + "/.jpg$/", + "/.jar$/", + "/__generated__/" ] } ], @@ -121,13 +120,48 @@ "filename": "Artsy.xcodeproj/xcshareddata/xcschemes/Artsy Stickers.xcscheme", "hashed_secret": "6aabb8b02d2915e10a5e6335cfc9af08a6f3f708", "is_verified": false, - "line_number": 18 + "line_number": 18, + "is_secret": true }, { "type": "Hex High Entropy String", "filename": "Artsy.xcodeproj/xcshareddata/xcschemes/Artsy Stickers.xcscheme", "hashed_secret": "15a5431bd32fc0fb1dd2bac526fb16d1494753be", "is_verified": false, + "line_number": 32, + "is_secret": true + } + ], + "Artsy.xcodeproj/xcshareddata/xcschemes/Artsy.xcscheme": [ + { + "type": "Hex High Entropy String", + "filename": "Artsy.xcodeproj/xcshareddata/xcschemes/Artsy.xcscheme", + "hashed_secret": "15a5431bd32fc0fb1dd2bac526fb16d1494753be", + "is_verified": false, + "line_number": 17, + "is_secret": true + }, + { + "type": "Hex High Entropy String", + "filename": "Artsy.xcodeproj/xcshareddata/xcschemes/Artsy.xcscheme", + "hashed_secret": "f87e1924bd54ea6e4be7644acd24c98f01d2b6b1", + "is_verified": false, + "line_number": 75 + } + ], + "Artsy.xcodeproj/xcshareddata/xcschemes/ArtsyWidgetExtension.xcscheme": [ + { + "type": "Hex High Entropy String", + "filename": "Artsy.xcodeproj/xcshareddata/xcschemes/ArtsyWidgetExtension.xcscheme", + "hashed_secret": "13d6a129a5a32f1ca827175ad94f372ae4a4029f", + "is_verified": false, + "line_number": 18 + }, + { + "type": "Hex High Entropy String", + "filename": "Artsy.xcodeproj/xcshareddata/xcschemes/ArtsyWidgetExtension.xcscheme", + "hashed_secret": "15a5431bd32fc0fb1dd2bac526fb16d1494753be", + "is_verified": false, "line_number": 32 } ], @@ -140,20 +174,223 @@ "line_number": 6 } ], + "Artsy/View_Controllers/live_auctions_socket.json": [ + { + "type": "Hex High Entropy String", + "filename": "Artsy/View_Controllers/live_auctions_socket.json", + "hashed_secret": "8c33911cc724f80a2448f86385843e8e5fbc2deb", + "is_verified": false, + "line_number": 4 + }, + { + "type": "Hex High Entropy String", + "filename": "Artsy/View_Controllers/live_auctions_socket.json", + "hashed_secret": "1197074fe5b1688d89854feea51cdb81a82b0163", + "is_verified": false, + "line_number": 26 + }, + { + "type": "Hex High Entropy String", + "filename": "Artsy/View_Controllers/live_auctions_socket.json", + "hashed_secret": "546a28e2fb097ecf73f202e4d76d23fae06664e6", + "is_verified": false, + "line_number": 48 + }, + { + "type": "Hex High Entropy String", + "filename": "Artsy/View_Controllers/live_auctions_socket.json", + "hashed_secret": "cf2c190b800ef74fc211f29f5bde9d11b04513e9", + "is_verified": false, + "line_number": 70 + }, + { + "type": "Hex High Entropy String", + "filename": "Artsy/View_Controllers/live_auctions_socket.json", + "hashed_secret": "5fa1aaa871c4398160078e9a69bc182fe8459d20", + "is_verified": false, + "line_number": 92 + }, + { + "type": "Hex High Entropy String", + "filename": "Artsy/View_Controllers/live_auctions_socket.json", + "hashed_secret": "f56be5ff1f0fcb442ab5552de01ed4a0f33a6197", + "is_verified": false, + "line_number": 114, + "is_secret": true + }, + { + "type": "Hex High Entropy String", + "filename": "Artsy/View_Controllers/live_auctions_socket.json", + "hashed_secret": "70202e4a31db4efda93ac5da4467aee15955e757", + "is_verified": false, + "line_number": 136 + }, + { + "type": "Hex High Entropy String", + "filename": "Artsy/View_Controllers/live_auctions_socket.json", + "hashed_secret": "441a1efbb2f5fb389482a6e9714bbb1156940fb5", + "is_verified": false, + "line_number": 158 + }, + { + "type": "Hex High Entropy String", + "filename": "Artsy/View_Controllers/live_auctions_socket.json", + "hashed_secret": "2bb9e6a9d15129a5fa4906884daab855b574d14e", + "is_verified": false, + "line_number": 180 + }, + { + "type": "Hex High Entropy String", + "filename": "Artsy/View_Controllers/live_auctions_socket.json", + "hashed_secret": "2fb6535d257d8b802e76567e0b2d1e70a6c992d5", + "is_verified": false, + "line_number": 202 + }, + { + "type": "Hex High Entropy String", + "filename": "Artsy/View_Controllers/live_auctions_socket.json", + "hashed_secret": "2bbe2040d1b3cc7e668020597d24cca4ff1c9c94", + "is_verified": false, + "line_number": 234 + }, + { + "type": "Hex High Entropy String", + "filename": "Artsy/View_Controllers/live_auctions_socket.json", + "hashed_secret": "5413f91d254d6035fdd778aefe3ed908dfdc3f51", + "is_verified": false, + "line_number": 385 + }, + { + "type": "Hex High Entropy String", + "filename": "Artsy/View_Controllers/live_auctions_socket.json", + "hashed_secret": "84494ababe4e8e8616748bb5e63d94b8cabfb02e", + "is_verified": false, + "line_number": 407 + }, + { + "type": "Hex High Entropy String", + "filename": "Artsy/View_Controllers/live_auctions_socket.json", + "hashed_secret": "33b79b577ef868edd8c22430c399b8952baaf1ef", + "is_verified": false, + "line_number": 429 + }, + { + "type": "Hex High Entropy String", + "filename": "Artsy/View_Controllers/live_auctions_socket.json", + "hashed_secret": "8e1dd297b57bbd9d31c0248274fca83ca7a37c20", + "is_verified": false, + "line_number": 451 + }, + { + "type": "Hex High Entropy String", + "filename": "Artsy/View_Controllers/live_auctions_socket.json", + "hashed_secret": "6db41c8b237d299690cc2e0a4f7e8262a5db86d7", + "is_verified": false, + "line_number": 473 + }, + { + "type": "Hex High Entropy String", + "filename": "Artsy/View_Controllers/live_auctions_socket.json", + "hashed_secret": "6c3d2a4ac9e1d85285af7bc1dd78c006296f17a4", + "is_verified": false, + "line_number": 495 + }, + { + "type": "Hex High Entropy String", + "filename": "Artsy/View_Controllers/live_auctions_socket.json", + "hashed_secret": "081249a9ea03533a59b18bdc677c9c5916d5c751", + "is_verified": false, + "line_number": 517 + }, + { + "type": "Hex High Entropy String", + "filename": "Artsy/View_Controllers/live_auctions_socket.json", + "hashed_secret": "65ce5f5dd82e1879cfaf2f9b7c77574ed8cbb811", + "is_verified": false, + "line_number": 539 + }, + { + "type": "Hex High Entropy String", + "filename": "Artsy/View_Controllers/live_auctions_socket.json", + "hashed_secret": "cfc760c8009779fd188ee5858206ea49552a9878", + "is_verified": false, + "line_number": 561 + }, + { + "type": "Hex High Entropy String", + "filename": "Artsy/View_Controllers/live_auctions_socket.json", + "hashed_secret": "b695bb92e86201d04c75ef5c71337934a7c9a63e", + "is_verified": false, + "line_number": 583 + }, + { + "type": "Hex High Entropy String", + "filename": "Artsy/View_Controllers/live_auctions_socket.json", + "hashed_secret": "05743cb98cadf74cce41f74650b7a32e655b6212", + "is_verified": false, + "line_number": 605 + }, + { + "type": "Hex High Entropy String", + "filename": "Artsy/View_Controllers/live_auctions_socket.json", + "hashed_secret": "a675752cd43399fa57679f18e0f96df6e19dfd47", + "is_verified": false, + "line_number": 627 + }, + { + "type": "Hex High Entropy String", + "filename": "Artsy/View_Controllers/live_auctions_socket.json", + "hashed_secret": "2ad401dfee2508f7293eda83bd7ca9fe993ad7aa", + "is_verified": false, + "line_number": 649 + } + ], + "ArtsyWidget/Fixtures.swift": [ + { + "type": "Hex High Entropy String", + "filename": "ArtsyWidget/Fixtures.swift", + "hashed_secret": "82c71809ac7630dec40d356c729955b7f4f0a23c", + "is_verified": false, + "line_number": 5 + }, + { + "type": "Hex High Entropy String", + "filename": "ArtsyWidget/Fixtures.swift", + "hashed_secret": "2e529c680f8ffeed9cba698fe2539a42e6a1e7fa", + "is_verified": false, + "line_number": 12 + }, + { + "type": "Hex High Entropy String", + "filename": "ArtsyWidget/Fixtures.swift", + "hashed_secret": "79c7825718a30565467f7ddd1075ba61c86e3374", + "is_verified": false, + "line_number": 19 + }, + { + "type": "Hex High Entropy String", + "filename": "ArtsyWidget/Fixtures.swift", + "hashed_secret": "98d980009dbf0506036553c0a3957171c321f33a", + "is_verified": false, + "line_number": 26 + } + ], "Artsy_Tests/View_Controller_Tests/Live_Auction/FakeSalesPerson.swift": [ { "type": "JSON Web Token", "filename": "Artsy_Tests/View_Controller_Tests/Live_Auction/FakeSalesPerson.swift", "hashed_secret": "54731282ed8341e2fd396060413d4bb168b5f99c", "is_verified": false, - "line_number": 78 + "line_number": 78, + "is_secret": false }, { "type": "JSON Web Token", "filename": "Artsy_Tests/View_Controller_Tests/Live_Auction/FakeSalesPerson.swift", "hashed_secret": "a5bb30a4f5e2458b4bab509514010ff3fefdac96", "is_verified": false, - "line_number": 82 + "line_number": 82, + "is_secret": false }, { "type": "JSON Web Token", @@ -163,6 +400,31 @@ "line_number": 86 } ], + "android/app/build.gradle": [ + { + "type": "Hex High Entropy String", + "filename": "android/app/build.gradle", + "hashed_secret": "91f71e355c66ef5ff819b35c867d4d28b9a8c469", + "is_verified": false, + "line_number": 151 + } + ], + "src/lib/Scenes/Artwork/Components/CommercialInformation.tests.tsx": [ + { + "type": "Hex High Entropy String", + "filename": "src/lib/Scenes/Artwork/Components/CommercialInformation.tests.tsx", + "hashed_secret": "ead2477c86563400ffd2577338c6c83ae48aeb5a", + "is_verified": false, + "line_number": 291 + }, + { + "type": "Hex High Entropy String", + "filename": "src/lib/Scenes/Artwork/Components/CommercialInformation.tests.tsx", + "hashed_secret": "f8c1197cdd5e93fc4ff721cd4417cad3b2fbd047", + "is_verified": false, + "line_number": 302 + } + ], "src/lib/Scenes/Consignments/fixtures/places.json": [ { "type": "Base64 High Entropy String", @@ -200,6 +462,31 @@ "line_number": 161 } ], + "src/lib/Scenes/MyCollection/Screens/ArtworkForm/MyCollectionArtworkForm.tests.tsx": [ + { + "type": "Hex High Entropy String", + "filename": "src/lib/Scenes/MyCollection/Screens/ArtworkForm/MyCollectionArtworkForm.tests.tsx", + "hashed_secret": "b5b44d59e3036fde34acae6a4ac3a669e27496ab", + "is_verified": false, + "line_number": 423 + } + ], + "src/lib/Scenes/MyCollection/utils/randomMyCollectionArtwork.ts": [ + { + "type": "Hex High Entropy String", + "filename": "src/lib/Scenes/MyCollection/utils/randomMyCollectionArtwork.ts", + "hashed_secret": "3ab18e9e12e053d935e40699910af0f0ff95b41a", + "is_verified": false, + "line_number": 19 + }, + { + "type": "Hex High Entropy String", + "filename": "src/lib/Scenes/MyCollection/utils/randomMyCollectionArtwork.ts", + "hashed_secret": "42079fa93fed2bbfcf020877932d355fd854f19f", + "is_verified": false, + "line_number": 20 + } + ], "src/lib/Scenes/Onboarding/OnboardingLogin.tsx": [ { "type": "Secret Keyword", @@ -272,5 +559,5 @@ } ] }, - "generated_at": "2022-01-27T20:09:02Z" + "generated_at": "2022-02-03T10:18:41Z" } diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index dac97702a1b..ab6fd80b0f6 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -2,6 +2,12 @@ ## Installation Issues +- Commit failed with: "ERROR: Potential secrets about to be committed to git repo!" + +This happens when you try to commit some code that looks like a secret, a key, a token, etc. +Make sure what you are committing has no sensitive data in it. +If you are sure is it _not_ sensitive data, then you can add an inline comment containing `pragma: allowlist secret`, to signify it is ok to commit. Then try to commit again, and it will work this time. + - Failed `./scripts/setup-env-for-artsy` with ``` @@ -9,7 +15,7 @@ yarn install ``` -make sure you are on proper node version and then do a +Make sure you are on proper node version and then do a ``` yarn install diff --git a/package.json b/package.json index c82a964d884..224a1749e67 100644 --- a/package.json +++ b/package.json @@ -67,8 +67,12 @@ "build-storybook": "build-storybook", "prestart-storybook": "yarn storybook-load-new", "start-storybook": "STORYBOOK=1 yarn start", - "detect-secrets-staged": "scripts/detect-secrets-staged", - "detect-secrets-all": "scripts/detect-secrets-all", + "secrets:audit": "detect-secrets audit .secrets.baseline", + "secrets:check:staged": "scripts/secrets-check-staged", + "secrets:check:all": "scripts/secrets-check-all", + "secrets:add-from:staged": "scripts/secrets-add-staged", + "secrets:add-from:all": "scripts/secrets-add-all", + "secrets:generate!": "scripts/secrets-generate-baseline", "update-metaphysics": "node scripts/update-metaphysics.js" }, "repository": { @@ -247,7 +251,6 @@ "danger": "10.7.1", "danger-plugin-jest": "1.3.0", "dedent": "0.7.0", - "detect-secrets": "1.0.6", "dotenv": "8.2.0", "enzyme": "3.9.0", "enzyme-adapter-react-16": "1.13.0", diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 3815e6673e2..00000000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -detect-secrets >= 1.1 diff --git a/scripts/detect-secrets-all b/scripts/detect-secrets-all deleted file mode 100755 index 8014932a2c7..00000000000 --- a/scripts/detect-secrets-all +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - - -yarn detect-secrets-launcher --baseline .secrets.baseline --exclude-secrets '[a-fA-F0-9]{24}' --exclude-lines 'W/"[!#-\x7E]*"' $(git ls-files | grep -v stickerpack) || git add .secrets.baseline diff --git a/scripts/detect-secrets-generate-baseline b/scripts/detect-secrets-generate-baseline deleted file mode 100755 index f1ca6f4bc1e..00000000000 --- a/scripts/detect-secrets-generate-baseline +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - - -npx detect-secrets scan > .secrets.baseline diff --git a/scripts/detect-secrets-staged b/scripts/detect-secrets-staged deleted file mode 100755 index d5179c47b21..00000000000 --- a/scripts/detect-secrets-staged +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - - -yarn detect-secrets-launcher --baseline .secrets.baseline --exclude-secrets '[a-fA-F0-9]{24}' --exclude-lines 'W/"[!#-\x7E]*"' $(git diff --staged --name-only | grep -v stickerpack) || git add .secrets.baseline diff --git a/scripts/install b/scripts/install index 4cb21bf72e2..4256610228c 100755 --- a/scripts/install +++ b/scripts/install @@ -3,6 +3,6 @@ set -euxo pipefail bundle check || bundle install +brew install detect_secrets # using brew version. read more here: https://github.com/artsy/homebrew-formulas/pull/13 yarn install yarn pod-install -pip install -r requirements.txt diff --git a/scripts/secrets-add-all b/scripts/secrets-add-all new file mode 100755 index 00000000000..aa62bc0fed6 --- /dev/null +++ b/scripts/secrets-add-all @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# This script runs to add secrets from all repo files to the baseline file. + +git ls-files \ +| tr '\n' '\0' \ +| xargs -0 detect-secrets scan \ + --exclude-files /\.lock$/ `# ignore lock files, they are large and full of hashes` \ + --exclude-files /\.png$/ `# ignore image files` \ + --exclude-files /\.webp$/ `# ignore image files` \ + --exclude-files /\.jpg$/ `# ignore image files` \ + --exclude-files /\.jar$/ `# ignore jar files` \ + --exclude-files /__generated__/ `# ignore relay generated files` \ + --baseline .secrets.baseline diff --git a/scripts/secrets-add-staged b/scripts/secrets-add-staged new file mode 100755 index 00000000000..f97003400a5 --- /dev/null +++ b/scripts/secrets-add-staged @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# This script runs to add secrets from staged files to the baseline file. + +git diff --staged --name-only \ +| tr '\n' '\0' \ +| xargs -0 detect-secrets scan \ + --exclude-files /\.lock$/ `# ignore lock files, they are large and full of hashes` \ + --exclude-files /\.png$/ `# ignore image files` \ + --exclude-files /\.webp$/ `# ignore image files` \ + --exclude-files /\.jpg$/ `# ignore image files` \ + --exclude-files /\.jar$/ `# ignore jar files` \ + --exclude-files /__generated__/ `# ignore relay generated files` \ + --baseline .secrets.baseline diff --git a/scripts/secrets-check-all b/scripts/secrets-check-all new file mode 100755 index 00000000000..699d55de1e4 --- /dev/null +++ b/scripts/secrets-check-all @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# This script runs to make sure all repo files don't contain secrets. + +git ls-files \ +| tr '\n' '\0' \ +| xargs -0 detect-secrets-hook \ + --baseline .secrets.baseline \ + --verbose diff --git a/scripts/secrets-check-staged b/scripts/secrets-check-staged new file mode 100755 index 00000000000..eb59dbb9ae2 --- /dev/null +++ b/scripts/secrets-check-staged @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# This script runs to make sure staged files don't contain secrets. + +git diff --staged --name-only \ +| tr '\n' '\0' \ +| xargs -0 detect-secrets-hook \ + --baseline .secrets.baseline \ + --verbose diff --git a/scripts/secrets-generate-baseline b/scripts/secrets-generate-baseline new file mode 100755 index 00000000000..2e8763289e7 --- /dev/null +++ b/scripts/secrets-generate-baseline @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# This script runs a command to generate a baseline of secrets. + +detect-secrets scan \ + --exclude-files /\.lock$/ `# ignore lock files, they are large and full of hashes` \ + --exclude-files /\.png$/ `# ignore image files` \ + --exclude-files /\.webp$/ `# ignore image files` \ + --exclude-files /\.jpg$/ `# ignore image files` \ + --exclude-files /\.jar$/ `# ignore jar files` \ + --exclude-files /__generated__/ `# ignore relay generated files` \ + > .secrets.baseline diff --git a/yarn.lock b/yarn.lock index f4afe984872..5774884ee66 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8864,14 +8864,6 @@ detect-port@^1.3.0: address "^1.0.1" debug "^2.6.0" -detect-secrets@1.0.6: - version "1.0.6" - resolved "https://registry.yarnpkg.com/detect-secrets/-/detect-secrets-1.0.6.tgz#ce6f46840e666f7d396ffb9b458c29ce9fdb5afd" - integrity sha512-bAEmXtMJNS/By/TCg9uSW9Sp0V1Z0N+uwlQWFUMbCVri5Yq5rM8gVs+2zzNIjNOy36o5kANZRrMc+22Zf6eRFQ== - dependencies: - debug "^4.1.0" - which "^1.3.1" - devtools-protocol@0.0.847576: version "0.0.847576" resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.847576.tgz#2f201bfb68aa9ef4497199fbd7f5d5dfde3b200b"