Merge pull request awslabs#18 from awslabs/scan-history-staging-branch

Add support for scan history (aggregated and rebased w/master)

README.rst

@@ -12,6 +12,7 @@ Synopsis
 ::
 
     git secrets --scan [-r|--recursive] [--cached] [--no-index] [--untracked] [<files>...]
+    git secrets --scan-history
     git secrets --install [-f|--force] [<target-directory>]
     git secrets --list [--global]
     git secrets --add [-a|--allowed] [-l|--literal] [--global] <pattern>
@@ -83,6 +84,13 @@ Each of these options must appear first on the command line.
     a colon, and then the line of text that matched. If no files are provided,
     all files returned by ``git ls-files`` are scanned.
 
+``--scan-history``
+    Scans repository including all revisions. When a file contains a secret, the
+    matched text from the file being scanned will be written to stdout and the
+    script will exit with a non-zero RC. Each matched line will be written with
+    the name of the file that matched, a colon, the line number that matched,
+    a colon, and then the line of text that matched.
+
 ``--list``
     Lists the git-secrets configuration for the current repo or in the global
     git config.

git-secrets

@@ -14,6 +14,7 @@
 
 NONGIT_OK=1 OPTIONS_SPEC="\
 git secrets --scan [-r|--recursive] [--cached] [--no-index] [--untracked] [<files>...]
+git secrets --scan-history
 git secrets --install [-f|--force] [<target-directory>]
 git secrets --list [--global]
 git secrets --add [-a|--allowed] [-l|--literal] [--global] <pattern>
@@ -22,6 +23,7 @@ git secrets --register-aws [--global]
 git secrets --aws-provider [<credentials-file>]
 --
 scan Scans <files> for prohibited patterns
+scan-history Scans repo for prohibited patterns
 install Installs git hooks for Git repository or Git template directory
 list Lists secret patterns
 add Adds a prohibited or allowed pattern, ensuring to de-dupe with existing patterns
@@ -59,26 +61,69 @@ load_allowed() {
   fi
 }
 
+# load patterns and combine them with |
+load_combined_patterns() {
+  local patterns=$(load_patterns)
+  local combined_patterns=''
+  for pattern in $patterns
+  do
+    combined_patterns=${combined_patterns}${pattern}"|"
+  done
+  combined_patterns=${combined_patterns%?}
+  echo $combined_patterns
+}
+
+# Scans files or a repo using patterns.
 scan() {
-  local files="$1" action='skip' patterns=$(load_patterns)
-  local allowed=$(load_allowed)
-  # No need to scan anything if there are no prohibited patterns.
-  [ -z "${patterns}" ] && return 0
-  # Build up the options to use when scanning with git-grep.
-  local options=""
+  local files="$1" options=""
   [ "${SCAN_CACHED}" == 1 ] && options+="--cached"
   [ "${SCAN_UNTRACKED}" == 1 ] && options+=" --untracked"
   [ "${SCAN_NO_INDEX}" == 1 ] && options+=" --no-index"
   # Scan using git-grep if there are no files or if git options are applied.
   if [ -z "${files}" ] || [ ! -z "${options}" ]; then
-    # Note: Passing an empty list of files will scan the entire repo.
-    output=$(GREP_OPTIONS= LC_ALL=C git grep -nwHE ${options} "${patterns}" ${files})
+    output=$(git_grep $options $files)
   else
-    # -r only applies when file paths are provided.
-    [ "${RECURSIVE}" -eq 1 ] && action="recurse"
-    output=$(GREP_OPTIONS= LC_ALL=C grep -d $action -nwHE "${patterns}" $files)
+    output=$(regular_grep $files)
   fi
-  local status=$?
+  process_output $? "${output}"
+}
+
+# Scans through history using patterns
+scan_history() {
+  # git log does not support multiple patterns, so we need to combine them
+  local combined_patterns=$(load_combined_patterns)
+  [ -z "${combined_patterns}" ] && return 0
+  # Looks for differences matching the patterns, reduces the number of revisions to scan
+  local to_scan=$(git log --all -G"${combined_patterns}" --pretty=%H)
+  # Scan through revisions with findings to normalize output
+  output=$(git_grep "" "${to_scan}")
+  process_output $? "${output}"
+}
+
+# Performs a git-grep, taking into account patterns and options.
+# Note: this function returns 1 on success, 0 on error.
+git_grep() {
+  local options="$1"; shift
+  local files=$@ patterns=$(load_patterns)
+  [ -z "${patterns}" ] && return 1
+  GREP_OPTIONS= LC_ALL=C git grep -nwHE ${options} "${patterns}" $@
+}
+
+# Performs a regular grep, taking into account patterns and recursion.
+# Note: this function returns 1 on success, 0 on error.
+regular_grep() {
+  local files=$@ patterns=$(load_patterns) action='skip'
+  [ -z "${patterns}" ] && return 1
+  [ "${RECURSIVE}" -eq 1 ] && action="recurse"
+  GREP_OPTIONS= LC_ALL=C grep -d "${action}" -nwHE "${patterns}" $@
+}
+
+# Process the given status ($1) and output variables ($2).
+# Takes into account allowed patterns, and if a bad match is found,
+# prints an error message and exits 1.
+process_output() {
+  local status="$1" output="$2"
+  local allowed=$(load_allowed)
   case "$status" in
     0)
       [ -z "${allowed}" ] && echo "${output}" && return 1
@@ -91,8 +136,11 @@ scan() {
   esac
 }
 
-scan_or_die() {
-  scan "$@" && exit 0
+# Calls the given scanning function at $1, shifts, and passes to it $@.
+# Exit 0 if success, otherwise exit 1 with error message.
+scan_with_fn_or_die() {
+  local fn="$1"; shift
+  $fn "$@" && exit 0
   echo
   echo "[ERROR] Matched one or more prohibited patterns"
   echo
@@ -108,7 +156,7 @@ scan_or_die() {
 
 # Scans a commit message, passed in the path to a file.
 commit_msg_hook() {
-  scan_or_die "$1"
+  scan_with_fn_or_die "scan" "$1"
 }
 
 # Scans all files that are about to be committed.
@@ -118,7 +166,7 @@ pre_commit_hook() {
   # Diff against HEAD if this is not the first commit in the repo.
   git rev-parse --verify HEAD >/dev/null 2>&1 && rev="HEAD"
   # Filter out deleted files using --diff-filter
-  scan_or_die "$(git diff-index --diff-filter 'ACMU' --name-only --cached $rev --)"
+  scan_with_fn_or_die "scan" "$(git diff-index --diff-filter 'ACMU' --name-only --cached $rev --)"
 }
 
 # Determines if merging in a commit will introduce tainted history.
@@ -129,7 +177,7 @@ prepare_commit_msg_hook() {
       local sha="${git_head##*=}"           # Get just the SHA
       local branch=$(git symbolic-ref HEAD) # e.g. refs/heads/master
       local dest="${branch#refs/heads/}"    # cut out "refs/heads"
-      git log "${dest}".."${sha}" -p | scan_or_die -
+      git log "${dest}".."${sha}" -p | scan_with_fn_or_die "scan" -
       ;;
   esac
 }
@@ -270,7 +318,8 @@ case "${COMMAND}" in
       add_config "secrets.patterns" "$1"
     fi
     ;;
-  --scan) scan_or_die "$@" ;;
+  --scan) scan_with_fn_or_die "scan" "$@" ;;
+  --scan-history) scan_with_fn_or_die "scan_history" "$@" ;;
   --list)
     if [ "${GLOBAL}" -eq 1 ]; then
       git config --global --get-regex secrets.*

git-secrets.1

@@ -40,6 +40,7 @@ git repository.
 .nf
 .ft C
 git secrets \-\-scan [\-r|\-\-recursive] [\-\-cached] [\-\-no\-index] [\-\-untracked] [<files>...]
+git secrets \-\-scan\-history
 git secrets \-\-install [\-f|\-\-force] [<target\-directory>]
 git secrets \-\-list [\-\-global]
 git secrets \-\-add [\-a|\-\-allowed] [\-l|\-\-literal] [\-\-global] <pattern>
@@ -127,6 +128,13 @@ the name of the file that matched, a colon, the line number that matched,
 a colon, and then the line of text that matched. If no files are provided,
 all files returned by \fBgit ls\-files\fP are scanned.
 .TP
+.B \fB\-\-scan\-history\fP
+Scans repository including all revisions. When a file contains a secret, the
+matched text from the file being scanned will be written to stdout and the
+script will exit with a non\-zero RC. Each matched line will be written with
+the name of the file that matched, a colon, the line number that matched,
+a colon, and then the line of text that matched.
+.TP
 .B \fB\-\-list\fP
 Lists the git\-secrets configuration for the current repo or in the global
 git config.

test/git-secrets.bats

@@ -36,6 +36,42 @@ load test_helper
   [ $status -eq 1 ]
 }
 
+@test "Scans all files including history" {
+  setup_bad_repo
+  repo_run git-secrets --scan-history
+  [ $status -eq 1 ]
+}
+
+@test "Scans all files when no file provided with secret in history" {
+  setup_bad_repo_history
+  repo_run git-secrets --scan
+  [ $status -eq 0 ]
+}
+
+@test "Scans all files including history with secret in history" {
+  setup_bad_repo_history
+  repo_run git-secrets --scan-history
+  [ $status -eq 1 ]
+}
+
+@test "Scans history with secrets distributed among branches in history" {
+  cd $TEST_REPO
+  echo '@todo' > $TEST_REPO/history_failure.txt
+  git add -A
+  git commit -m "Testing history"
+  echo 'todo' > $TEST_REPO/history_failure.txt
+  git add -A
+  git commit -m "Testing history"
+  git checkout -b testbranch
+  echo '@todo' > $TEST_REPO/history_failure.txt
+  git add -A
+  git commit -m "Testing history"
+  git checkout master
+  cd -
+  repo_run git-secrets --scan-history
+  [ $status -eq 1 ]
+}
+
 @test "Scans recursively" {
   setup_bad_repo
   mkdir -p $TEST_REPO/foo/bar/baz

test/test_helper.bash

@@ -57,6 +57,17 @@ setup_bad_repo() {
   cd -
 }
 
+# Creates a repo that should fail
+setup_bad_repo_history() {
+  cd $TEST_REPO
+  echo '@todo' > $TEST_REPO/history_failure.txt
+  git add -A
+  git commit -m "Testing history"
+  echo 'todo' > $TEST_REPO/history_failure.txt
+  git add -A
+  cd -
+}
+
 # Creates a repo that does not fail
 setup_good_repo() {
   cd $TEST_REPO