Merge pull request #13 from awslabs/staged-pre-commit-scan

Ensure pre-commit hook scans staged files

README.rst

@@ -11,7 +11,7 @@ Synopsis
 
 ::
 
-    git secrets --scan [-r|--recursive] [<files>...]
+    git secrets --scan [-r|--recursive] [--cached] [--no-index] [--untracked] [<files>...]
     git secrets --install [-f|--force] [<target-directory>]
     git secrets --list [--global]
     git secrets --add [-a|--allowed] [-l|--literal] [--global] <pattern>
@@ -197,6 +197,19 @@ Options for ``--scan``
     directory will be scanned. If ``-r`` is not provided, directories will be
     ignored.
 
+    ``-r`` cannot be used alongside ``--cached``, ``--no-index``, or
+    ``--untracked``.
+
+``--cached``
+    Searches blobs registered in the index file.
+
+``--no-index``
+    Searches files in the current directory that is not managed by Git.
+
+``--untracked``
+    In addition to searching in the tracked files in the working tree,
+    ``--scan`` also in untracked files.
+
 ``<files>...``
     The path to one or more files on disk to scan for secrets.
 

git-secrets

@@ -13,11 +13,11 @@
 # permissions and limitations under the License.
 
 NONGIT_OK=1 OPTIONS_SPEC="\
-git secrets --scan [-r|--recursive] [<files>...]
+git secrets --scan [-r|--recursive] [--cached] [--no-index] [--untracked] [<files>...]
 git secrets --install [-f|--force] [<target-directory>]
 git secrets --list [--global]
 git secrets --add [-a|--allowed] [-l|--literal] [--global] <pattern>
-git secrets --add-provider <command> [arguments...]
+git secrets --add-provider [--global] <command> [arguments...]
 git secrets --register-aws [--global]
 git secrets --aws-provider [<credentials-file>]
 --
@@ -29,6 +29,9 @@ add-provider Adds a secret provider that when called outputs secret patterns on
 aws-provider Secret provider that outputs credentials found in an ini file
 register-aws Adds common AWS patterns to the git config and scans for ~/.aws/credentials
 r,recursive --scan scans directories recursively
+cached --scan scans searches blobs registered in the index file
+no-index --scan searches files in the current directory that is not managed by Git
+untracked In addition to searching in the tracked files in the working tree, --scan also in untracked files
 f,force --install overwrites hooks if the hook already exists
 l,literal --add and --add-allowed patterns are escaped so that they are literal
 a,allowed --add adds an allowed pattern instead of a prohibited pattern
@@ -51,9 +54,17 @@ load_patterns() {
 scan() {
   local files="$1" action='skip' patterns=$(load_patterns)
   local allowed=$(git config --get-all secrets.allowed)
+  # No need to scan anything if there are no prohibited patterns.
   [ -z "${patterns}" ] && return 0
-  if [ -z "${files}" ]; then
-    output=$(GREP_OPTIONS= LC_ALL=C git grep -nwHE "${patterns}")
+  # Build up the options to use when scanning with git-grep.
+  local options=""
+  [ "${SCAN_CACHED}" == 1 ] && options+="--cached"
+  [ "${SCAN_UNTRACKED}" == 1 ] && options+=" --untracked"
+  [ "${SCAN_NO_INDEX}" == 1 ] && options+=" --no-index"
+  # Scan using git-grep if there are no files or if git options are applied.
+  if [ -z "${files}" ] || [ ! -z "${options}" ]; then
+    # Note: Passing an empty list of files will scan the entire repo.
+    output=$(GREP_OPTIONS= LC_ALL=C git grep -nwHE ${options} "${patterns}" ${files})
   else
     # -r only applies when file paths are provided.
     [ "${RECURSIVE}" -eq 1 ] && action="recurse"
@@ -92,6 +103,7 @@ commit_msg_hook() {
 
 # Scans all files that are about to be committed.
 pre_commit_hook() {
+  SCAN_CACHED=1
   local file found_match=0 rev="4b825dc642cb6eb9a060e54bf8d69288fbee4904"
   # Diff against HEAD if this is not the first commit in the repo.
   git rev-parse --verify HEAD >/dev/null 2>&1 && rev="HEAD"
@@ -173,21 +185,66 @@ aws_provider() {
   fi
 }
 
+# Ensures that the command is what was expected for an option.
+assert_option_for_command() {
+  local expected_command="$1"
+  local option_name="$2"
+  if [ "${COMMAND}" != "${expected_command}" ]; then
+    die "${option_name} can only be supplied with the ${expected_command} subcommand"
+  fi
+}
+
 declare COMMAND="$1" FORCE=0 RECURSIVE=0 LITERAL=0 GLOBAL=0 ALLOWED=0
+declare SCAN_CACHED=0 SCAN_NO_INDEX=0 SCAN_UNTRACKED=0
+
 # Shift off the command name
 shift 1
 while [ "$#" -ne 0 ]; do
   case "$1" in
-    -f) FORCE=1 ;;
-    -r) RECURSIVE=1 ;;
-    -a) ALLOWED=1 ;;
-    -l) LITERAL=1 ;;
+    -f)
+      assert_option_for_command "--install" "-f|--force"
+      FORCE=1
+      ;;
+    -r)
+      assert_option_for_command "--scan" "-r|--recursive"
+      RECURSIVE=1
+      ;;
+    -a)
+      assert_option_for_command "--add" "-a|--allowed"
+      ALLOWED=1
+      ;;
+    -l)
+      assert_option_for_command "--add" "-l|--literal"
+      LITERAL=1
+      ;;
+    --cached)
+      assert_option_for_command "--scan" "--cached"
+      SCAN_CACHED=1
+      ;;
+    --no-index)
+      assert_option_for_command "--scan" "--no-index"
+      SCAN_NO_INDEX=1
+      ;;
+    --untracked)
+      assert_option_for_command "--scan" "--untracked"
+      SCAN_UNTRACKED=1
+      ;;
     --global) GLOBAL=1 ;;
     --) shift; break ;;
   esac
   shift
 done
 
+# Ensure that recursive is not applied with mutually exclusive options.
+if [ "${RECURSIVE}" -eq 1 ]; then
+  if [ "${SCAN_CACHED}" ] \
+      || [ "${SCAN_NO_INDEX}" ] \
+      || [ "${SCAN_UNTRACKED}" ];
+  then
+    die "-r|--recursive cannot be supplied with --cached, --no-index, or --untracked"
+  fi
+fi
+
 case "${COMMAND}" in
   -h|--help|--) "$0" -h; exit 0 ;;
   --add-provider) add_config "secrets.providers" "$@" ;;

git-secrets.1

@@ -39,7 +39,7 @@ git repository.
 .sp
 .nf
 .ft C
-git secrets \-\-scan [\-r|\-\-recursive] [<files>...]
+git secrets \-\-scan [\-r|\-\-recursive] [\-\-cached] [\-\-no\-index] [\-\-untracked] [<files>...]
 git secrets \-\-install [\-f|\-\-force] [<target\-directory>]
 git secrets \-\-list [\-\-global]
 git secrets \-\-add [\-a|\-\-allowed] [\-l|\-\-literal] [\-\-global] <pattern>
@@ -74,6 +74,18 @@ make install
 .UNINDENT
 .UNINDENT
 .sp
+Or, installing with Homebrew (for OS X users).
+.INDENT 0.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+brew install git\-secrets
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.sp
 \fBWARNING:\fP
 .INDENT 0.0
 .INDENT 3.5
@@ -274,6 +286,19 @@ git secrets \-\-install \-f
 Scans the given files recursively. If a directory is encountered, the
 directory will be scanned. If \fB\-r\fP is not provided, directories will be
 ignored.
+.sp
+\fB\-r\fP cannot be used alongside \fB\-\-cached\fP, \fB\-\-no\-index\fP, or
+\fB\-\-untracked\fP\&.
+.TP
+.B \fB\-\-cached\fP
+Searches blobs registered in the index file.
+.TP
+.B \fB\-\-no\-index\fP
+Searches files in the current directory that is not managed by Git.
+.TP
+.B \fB\-\-untracked\fP
+In addition to searching in the tracked files in the working tree,
+\fB\-\-scan\fP also in untracked files.
 .TP
 .B \fB<files>...\fP
 The path to one or more files on disk to scan for secrets.

test/git-secrets.bats

@@ -227,3 +227,47 @@ load test_helper
   echo "$output" | grep -F 'foo baz bar'
   echo "$output" | grep -F 'bam'
 }
+
+@test "--recursive cannot be used with SCAN_*" {
+  repo_run git-secrets --scan -r --cached
+  [ $status -eq 1 ]
+  repo_run git-secrets --scan -r --no-index
+  [ $status -eq 1 ]
+  repo_run git-secrets --scan -r --untracked
+  [ $status -eq 1 ]
+}
+
+@test "-recursive can only be used with --scan" {
+  repo_run git-secrets --list -r
+  [ $status -eq 1 ]
+}
+
+@test "-f can only be used with --install" {
+  repo_run git-secrets --scan -f
+  [ $status -eq 1 ]
+}
+
+@test "-a can only be used with --add" {
+  repo_run git-secrets --scan -a
+  [ $status -eq 1 ]
+}
+
+@test "-l can only be used with --add" {
+  repo_run git-secrets --scan -l
+  [ $status -eq 1 ]
+}
+
+@test "--cached can only be used with --scan" {
+  repo_run git-secrets --list --cached
+  [ $status -eq 1 ]
+}
+
+@test "--no-index can only be used with --scan" {
+  repo_run git-secrets --list --no-index
+  [ $status -eq 1 ]
+}
+
+@test "--untracked can only be used with --scan" {
+  repo_run git-secrets --list --untracked
+  [ $status -eq 1 ]
+}

test/pre-commit.bats

@@ -12,6 +12,18 @@ load test_helper
   [ "${lines[2]}" == "failure2.txt:1:me" ]
 }
 
+@test "Scans staged files" {
+  cd $TEST_REPO
+  repo_run git-secrets --install $TEST_REPO
+  echo '@todo more stuff' > $TEST_REPO/data.txt
+  echo 'hi there' > $TEST_REPO/ok.txt
+  git add -A
+  echo 'fixed the working directory, but not staged' > $TEST_REPO/data.txt
+  run git commit -m 'Contents are bad not the message'
+  [ $status -eq 1 ]
+  [ "${lines[0]}" == "data.txt:1:@todo more stuff" ]
+}
+
 @test "Allows commits that do not match prohibited patterns" {
   setup_good_repo
   repo_run git-secrets --install $TEST_REPO