From 016fc55b9eb4371b6164075f732ee205790be0d0 Mon Sep 17 00:00:00 2001 From: Michael Dowling Date: Wed, 9 Dec 2015 22:31:44 -0700 Subject: [PATCH] Initial commit of git-secrets --- .travis.yml | 8 + CHANGELOG.md | 5 + LICENSE.txt | 208 +++++++ Makefile | 25 + NOTICE.txt | 6 + README.rst | 449 +++++++++++++++ git-secrets | 226 ++++++++ git-secrets.1 | 660 +++++++++++++++++++++++ test/bats/LICENSE | 20 + test/bats/bin/bats | 1 + test/bats/libexec/bats | 142 +++++ test/bats/libexec/bats-exec-suite | 55 ++ test/bats/libexec/bats-exec-test | 346 ++++++++++++ test/bats/libexec/bats-format-tap-stream | 165 ++++++ test/bats/libexec/bats-preprocess | 52 ++ test/commit-msg.bats | 18 + test/git-secrets.bats | 229 ++++++++ test/pre-commit.bats | 27 + test/prepare-commit-msg.bats | 33 ++ test/test_helper.bash | 57 ++ 20 files changed, 2732 insertions(+) create mode 100644 .travis.yml create mode 100644 CHANGELOG.md create mode 100644 LICENSE.txt create mode 100644 Makefile create mode 100644 NOTICE.txt create mode 100644 README.rst create mode 100755 git-secrets create mode 100644 git-secrets.1 create mode 100644 test/bats/LICENSE create mode 120000 test/bats/bin/bats create mode 100755 test/bats/libexec/bats create mode 100755 test/bats/libexec/bats-exec-suite create mode 100755 test/bats/libexec/bats-exec-test create mode 100755 test/bats/libexec/bats-format-tap-stream create mode 100755 test/bats/libexec/bats-preprocess create mode 100644 test/commit-msg.bats create mode 100644 test/git-secrets.bats create mode 100644 test/pre-commit.bats create mode 100644 test/prepare-commit-msg.bats create mode 100644 test/test_helper.bash diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..2593798 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,8 @@ +language: bash + +before_install: +- git config --global user.email "you@example.com" +- git config --global user.name "Your Name" + +script: +- make test diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..de8bae0 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +# CHANGELOG + +## 1.0.0 - 2015-12-10 + +* Initial release of ``git-secrets``. diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..de96b94 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,208 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Note: Other license terms may apply to certain, identified software files +contained within or distributed with the accompanying software if such terms +are included in the directory containing the accompanying software. Such other +license terms will then apply in lieu of the terms of the software license +above. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..2ddcda7 --- /dev/null +++ b/Makefile @@ -0,0 +1,25 @@ +PREFIX ?= /usr/local +MANPREFIX ?= "${PREFIX}/share/man/man1" + +help: + @echo "Please use \`make ' where is one of" + @echo " test to perform unit tests." + @echo " man to build the man file from README.rst" + @echo " install to install. Use PREFIX and MANPREFIX to customize." + +# We use bats for testing: https://github.com/sstephenson/bats +test: + test/bats/bin/bats test/ + +# The man page is completely derived from README.rst. Edits to +# README.rst require a rebuild of the man page. +man: + rst2man.py README.rst > git-secrets.1 + +install: + @mkdir -p ${DESTDIR}${MANPREFIX} + @mkdir -p ${DESTDIR}${PREFIX}/bin + @cp -f git-secrets ${DESTDIR}${PREFIX}/bin + @cp -f git-secrets.1 ${DESTDIR}${MANPREFIX} + +.PHONY: help test man diff --git a/NOTICE.txt b/NOTICE.txt new file mode 100644 index 0000000..a5e5da9 --- /dev/null +++ b/NOTICE.txt @@ -0,0 +1,6 @@ +git-secrets +Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. + +bats +This product bundles bats, which is available under a "MIT" license. +For details, see test/bats. diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..e366e43 --- /dev/null +++ b/README.rst @@ -0,0 +1,449 @@ +=========== +git-secrets +=========== + +Prevents you from committing passwords and other sensitive information to a +git repository. + + +Synopsis +-------- + +:: + + git secrets --scan [-r|--recursive] [...] + git secrets --install [-f|--force] [] + git secrets --list [--global] + git secrets --add [-a|--allowed] [-l|--literal] [--global] + git secrets --add-provider [--global] [arguments...] + git secrets --register-aws [--global] + git secrets --aws-provider [] + + +Description +----------- + +``git-secrets`` scans commits, commit messages, and ``--no-ff`` merges to +prevent adding secrets into your git repositories. If a commit, +commit message, or any commit in a ``--no-ff`` merge history matches one of +your configured prohibited regular expression patterns, then the commit is +rejected. + + +Installing git-secrets +~~~~~~~~~~~~~~~~~~~~~~ + +``git-secrets`` must be placed somewhere in your PATH so that it is picked up +by ``git`` when running ``git secrets``. You can use ``install`` target of the +provided Makefile to install ``git secrets`` and the man page. You can +customize the install path using the PREFIX and MANPREFIX variables. + +:: + + make install + +.. warning:: + + You're not done yet! You MUST install the git hooks for every repo that + you wish to use with ``git secrets --install``. + + +Options +------- + +Operation Modes +~~~~~~~~~~~~~~~ + +Each of these options must appear first on the command line. + +``--install`` + Installs hooks for a repository. Once the hooks are installed for a git + repository, commits and non-ff merges for that repository will be prevented + from committing secrets. + +``--scan`` + Scans one or more files for secrets. When a file contains a secret, the + matched text from the file being scanned will be written to stdout and the + script will exit with a non-zero RC. Each matched line will be written with + the name of the file that matched, a colon, the line number that matched, + a colon, and then the line of text that matched. If no files are provided, + all files returned by ``git ls-files`` are scanned. + +``--list`` + Lists the git-secrets configuration for the current repo or in the global + git config. + +``--add`` + Adds a prohibited or allowed pattern. + +``add-provider`` + Registers a secret provider. Secret providers are executables that when + invoked outputs prohibited patterns that ``git-secrets`` should treat as + prohibited. + +``--register-aws`` + Adds common AWS patterns to the git config and ensures that keys present + in ``~/.aws/credentials`` are not found in any commit. The following + checks are added: + + - AWS Access Key ID via ``[A-Z0-9]{20}`` + - AWS Secret Access Key assignments via ":" or "=" surrounded by optional + quotes + - AWS account ID assignments via ":" or "=" surrounded by optional quotes + - Allowed patterns for example AWS keys (``AKIAIOSFODNN7EXAMPLE`` and + ``wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY``) + - Enables using ``~/.aws/credentials`` to scan for known credentials. + + .. note:: + + While the patterns registered by this command should catch most + instances of AWS credentials, these patterns are **not** guaranteed to + catch them **all**. ``git-secrets`` should be used as an extra means of + insurance -- you still need to do your due diligence to ensure that you + do not commit credentials to a repository. + +``--aws-provider`` + Secret provider that outputs credentials found in an INI file. You can + optionally provide the path to an ini file. + + +Options for ``--install`` +~~~~~~~~~~~~~~~~~~~~~~~~~ + +``-f, --force`` + Overwrites existing hooks if present. + +```` + When provided, installs git hooks to the given directory. The current + directory is assumed if ```` is not provided. + + If the provided ```` is not in a Git repository, the + directory will be created and hooks will be placed in + ``/hooks``. This can be useful for creating Git template + directories using with ``git init --template ``. + + You can run ``git init`` on a repository that has already been initialized. + From the `git init documentation `_: + + From the git documentation: Running git init in an existing repository + is safe. It will not overwrite things that are already there. The + primary reason for rerunning git init is to pick up newly added + templates (or to move the repository to another place if + ``--separate-git-dir`` is given). + + The following git hooks are installed: + + 1. ``pre-commit``: Used to check if any of the files changed in the commit + use prohibited patterns. + 2. ``commit-msg``: Used to determine if a commit message contains a + prohibited patterns. + 3. ``prepare-commit-msg``: Used to determine if a merge commit will + introduce a history that contains a prohibited pattern at any point. + Please note that this hook is only invoked for non fast-forward merges. + + .. note:: + + Git only allows a single script to be executed per hook. If the + repository contains Debian style subdirectories like ``pre-commit.d`` + and ``commit-msg.d``, then the git hooks will be installed into these + directories, which assumes that you've configured the corresponding + hooks to execute all of the scripts found in these directories. If + these git subdirectories are not present, then the git hooks will be + installed to the git repo's ``.git/hooks`` directory. + + +Examples +^^^^^^^^ + +Install git hooks to the current directory:: + + cd /path/to/my/repository + git secrets --install + +Install git hooks to a repository other than the current directory:: + + git secrets --install -/path/to/my/repository + +Create a git template that has ``git-secrets`` installed, and then copy that +template into a git repository:: + + git secrets --install ~/.git-templates/git-secrets + git init --template ~/.git-templates/git-secrets + +Overwrite existing hooks if present:: + + git secrets --install -f + + +Options for ``--scan`` +~~~~~~~~~~~~~~~~~~~~~~ + +``-r, --recursive`` + Scans the given files recursively. If a directory is encountered, the + directory will be scanned. If ``-r`` is not provided, directories will be + ignored. + +``...`` + The path to one or more files on disk to scan for secrets. + + If no files are provided, all files returned by ``git ls-files`` are + scanned. + + +Examples +^^^^^^^^ + +Scan all files in the repo:: + + git secrets --scan + +Scans a single file for secrets:: + + git secrets --scan /path/to/file + +Scans a directory recursively for secrets:: + + git secrets --scan -r /path/to/directory + +Scans multiple files for secrets:: + + git secrets --scan /path/to/file /path/to/other/file + +You can scan by globbing:: + + git secrets --scan /path/to/directory/* + +Scan from stdin:: + + echo 'hello!' | git secrets --scan - + + +Options for ``--list`` +~~~~~~~~~~~~~~~~~~~~~~ + +``--global`` + Lists only git-secrets configuration in the global git config. + + +Options for ``--add`` +~~~~~~~~~~~~~~~~~~~~~ + +``--global`` + Adds patterns to the global git config + +``-l, --literal`` + Escapes special regular expression characters in the provided pattern so + that the pattern is searched for literally. + +``-a, --allowed`` + Mark the pattern as allowed instead of prohibited. Allowed patterns are + used to filter our false positives. + +```` + The regex pattern to search. + + +Examples +^^^^^^^^ + +Adds a prohibited pattern to the current repo:: + + git secrets --add '[A-Z0-9]{20}' + +Adds a prohibited pattern to the global git config:: + + git secrets --add --global '[A-Z0-9]{20}' + +Adds a string that is scanned for literally (``+`` is escaped):: + + git secrets --add --literal 'foo+bar' + +Add an allowed pattern:: + + git secrets --add -a 'allowed pattern' + + +Options for ``--register-aws`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``--global`` + Adds AWS specific configuration variables to the global git config. + + +Options for ``--aws-provider`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``[]`` + If provided, specifies the custom path to an INI file to scan. If not + provided, ``~/.aws/credentials`` is assumed. + + +Options for ``--add-provider`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``--global`` + Adds the provider to the global git config. + +```` + Provider command to invoke. When invoked the command is expected to write + prohibited patterns separated by new lines to stdout. Any extra arguments + provided are passed on to the command. + + +Examples +^^^^^^^^ + +Registers a secret provider with arguments:: + + git secrets --add-provider -- git secrets --aws-provider + +Cats secrets out of a file:: + + git secrets --add-provider -- cat /path/to/secret/file/patterns + + +Defining prohibited patterns +---------------------------- + +egrep compatible regular expressions are used to determine if a commit or +commit message contains any prohibited patterns. These regular expressions are +defined using the ``git config`` command. It is important to note that +different systems use different versions of egrep. For example, when running on +OS X, you will use a different version of egrep than when running on something +like Ubuntu (BSD vs GNU). + +You can add prohibited regular expression patterns to your git config using +``git secrets --add ``. + + +Ignoring false-positives +------------------------ + +Sometimes a regular expression might match false positives. For example, git +commit SHAs look a lot like AWS access keys. You can specify many different +regular expression patterns as false positives using the following command: + +:: + + git secrets --add --allowed 'my regex pattern' + +First, git-secrets will extract all lines from a file that contain a prohibited +match. Included in the matched results will be the full path to the name of +the file that was matched, followed ':', followed by the line number that was +matched, followed by the entire line from the file that was matched by a secret +pattern. Then, if you've defined allowed regular expressions, git-secrets will +check to see if all of the matched lines match at least one of your registered +allowed regular expressions. If all of the lines that were flagged as secret +are canceled out by an allowed match, then the subject text does not contain +any secrets. If any of the matched lines are not matched by an allowed regular +expression, then git-secrets will fail the commit/merge/message. + +.. important:: + + Just as it is a bad practice to add prohibited patterns that are too + greedy, it is also a bad practice to add allowed patterns that are too + forgiving. Be sure to test out your patterns using ad-hoc calls to + ``git secrets --scan $filename`` to ensure they are working as intended. + + +Secret providers +---------------- + +Sometimes you want to check for an exact pattern match against a set of known +secrets. For example, you might want to ensure that no credentials present in +``~/.aws/credentials`` ever show up in a commit. In these cases, it's better to +leave these secrets in one location rather than spread them out across git +repositories in git configs. You can use "secret providers" to fetch these +types of credentials. A secret provider is an executable that when invoked +outputs prohibited patterns separated by new lines. + +You can add secret providers using the ``--add-provider`` command:: + + git secrets --add-provider -- git secrets --aws-provider + +Notice the use of ``--``. This ensures that any arguments associated with the +provider are passed to the provider each time it is invoked when scanning +for secrets. + + +Example walkthrough +------------------- + +Let's take a look at an example. Given the following subject text (stored in +``/tmp/example``):: + + This is a test! + password=ex@mplepassword + password=****** + More test... + +And the following registered patterns: + +:: + + git config --add 'password\s*=\s*.+' + git config --add --allowed --literal 'ex@mplepassword' + +Running ``git secrets --scan /tmp/example``, the result will +result in the following error output:: + + /tmp/example:3:password=****** + + [ERROR] Matched prohibited pattern + + Possible mitigations: + - Mark false positives as allowed using: git config --add secrets.allowed ... + - List your configured patterns: git config --get-all secrets.patterns + - List your configured allowed patterns: git config --get-all secrets.allowed + - Use --no-verify if this is a one-time false positive + +Breaking this down, the prohibited pattern value of ``password\s*=\s*.+`` will +match the following lines:: + + /tmp/example:2:password=ex@mplepassword + /tmp/example:3:password=****** + +...But the first match will be filtered out due to the fact that it matches the +allowed regular expression of ``ex@mplepassword``. Because there is still a +remaining line that did not match, it is considered a secret. + +Because that matching lines are placed on lines that start with the filename +and line number (e.g., ``/tmp/example:3:...``), you can create allowed +patterns that take filenames and line numbers into account in the regular +expression. For example, you could whitelist an entire file using something +like:: + + git secrets --add --allowed '/tmp/example:.*' + git secrets --scan /tmp/example && echo $? + # Outputs: 0 + +Alternatively, you could whitelist a specific line number of a file if that +line is unlikely to change using something like the following: + +:: + + git secrets --add --allowed '/tmp/example:3:.*' + git secrets --scan /tmp/example && echo $? + # Outputs: 0 + +Keep this in mind when creating allowed patterns to ensure that your allowed +patterns are not inadvertantly matched due to the fact that the filename is +included in the subject text that allowed patterns are matched against. + + +Skipping validation +------------------- + +Use the ``--no-verify`` option in the event of a false-positive match in a +commit, merge, or commit message. This will skip the execution of the +git hook and allow you to make the commit or merge. + + +About +------ + +- Author: Michael Dowling +- Issue tracker: This project's source code and issue tracker can be found at + `https://github.com/awslabs/git-secrets `_ + +Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/git-secrets b/git-secrets new file mode 100755 index 0000000..8040f7d --- /dev/null +++ b/git-secrets @@ -0,0 +1,226 @@ +#!/usr/bin/env bash +# Copyright 2010-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://aws.amazon.com/apache2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +NONGIT_OK=1 OPTIONS_SPEC="\ +git secrets --scan [-r|--recursive] [...] +git secrets --install [-f|--force] [] +git secrets --list [--global] +git secrets --add [-a|--allowed] [-l|--literal] [--global] +git secrets --add-provider [arguments...] +git secrets --register-aws [--global] +git secrets --aws-provider [] +-- +scan Scans for prohibited patterns +install Installs git hooks for Git repository or Git template directory +list Lists secret patterns +add Adds a prohibited or allowed pattern, ensuring to de-dupe with existing patterns +add-provider Adds a secret provider that when called outputs secret patterns on new lines +aws-provider Secret provider that outputs credentials found in an ini file +register-aws Adds common AWS patterns to the git config and scans for ~/.aws/credentials +r,recursive --scan scans directories recursively +f,force --install overwrites hooks if the hook already exists +l,literal --add and --add-allowed patterns are escaped so that they are literal +a,allowed --add adds an allowed pattern instead of a prohibited pattern +global Uses the --global git config +commit_msg_hook* commit-msg hook (internal only) +pre_commit_hook* pre-commit hook (internal only) +prepare_commit_msg_hook* prepare-commit-msg hook (internal only)" + +# Include the git setup script. This parses and normalized CLI arguments. +. $(git --exec-path)/git-sh-setup + +load_patterns() { + git config --get-all secrets.patterns + # Execute each provider and use their output to build up patterns + git config --get-all secrets.providers | while read cmd; do + echo "$($cmd)" + done +} + +scan() { + local files="$1" action='skip' patterns=$(load_patterns) + local allowed=$(git config --get-all secrets.allowed) + [ -z "${patterns}" ] && return 0 + [ "${RECURSIVE}" -eq 1 ] && action="recurse" + output=$(GREP_OPTIONS= LC_ALL=C grep -d $action -nwHE "${patterns}" $files) + local status=$? + case "$status" in + 0) + [ -z "${allowed}" ] && echo "${output}" && return 1 + # Determine with a negative grep if the found matches are allowed + echo "${output}" | GREP_OPTIONS= LC_ALL=C grep -Ev "${allowed}" \ + && return 1 || return 0 + ;; + 1) return 0 ;; + *) exit $status + esac +} + +scan_or_die() { + scan "$@" && exit 0 + echo + echo "[ERROR] Matched one or more prohibited patterns" + echo + echo "Possible mitigations:" + echo "- Mark false positives as allowed using: git config --add secrets.allowed ..." + echo "- List your configured patterns: git config --get-all secrets.patterns" + echo "- List your configured allowed patterns: git config --get-all secrets.allowed" + echo "- Use --no-verify if this is a one-time false positive" + exit 1 +} + +# Scans a commit message, passed in the path to a file. +commit_msg_hook() { + scan_or_die "$1" +} + +# Scans all files that are about to be committed. +pre_commit_hook() { + local file found_match=0 rev="4b825dc642cb6eb9a060e54bf8d69288fbee4904" + # Diff against HEAD if this is not the first commit in the repo. + git rev-parse --verify HEAD >/dev/null 2>&1 && rev="HEAD" + # Filter out deleted files using --diff-filter + scan_or_die "$(git diff-index --diff-filter 'ACMU' --name-only --cached $rev --)" +} + +# Determines if merging in a commit will introduce tainted history. +prepare_commit_msg_hook() { + case "$2,$3" in + merge,) + local git_head=$(env | grep GITHEAD) # e.g. GITHEAD_=release/1.43 + local sha="${git_head##*=}" # Get just the SHA + local branch=$(git symbolic-ref HEAD) # e.g. refs/heads/master + local dest="${branch#refs/heads/}" # cut out "refs/heads" + git log "${dest}".."${sha}" -p | scan_or_die - + ;; + esac +} + +install_hook() { + local path="$1" hook="$2" cmd="$3" dest + # Determines the approriate path for a hook to be installed + if [ -d "${path}/hooks/${hook}.d" ]; then + dest="${path}/hooks/${hook}.d/git-secrets" + else + dest="${path}/hooks/${hook}" + fi + [ -f "${dest}" ] && [ "${FORCE}" -ne 1 ] \ + && die "${dest} already exists. Use -f to force" + echo "#!/usr/bin/env bash" > "${dest}" + echo "git secrets --${cmd} -- \"\$@\"" >> "${dest}" + chmod +x "${dest}" + say "$(tput setaf 2)✓$(tput sgr 0) Installed ${hook} hook to ${dest}" +} + +install_all_hooks() { + install_hook "$1" "commit-msg" "commit_msg_hook" + install_hook "$1" "pre-commit" "pre_commit_hook" + install_hook "$1" "prepare-commit-msg" "prepare_commit_msg_hook" +} + +# Adds a git config pattern, ensuring to de-dupe +add_config() { + local key="$1"; shift + local value="$@" + if [ "${LITERAL}" -eq 1 ]; then + value=$(sed 's/[\.|$(){}?+*^]/\\&/g' <<< "${value}") + fi + if [ "${GLOBAL}" -eq 1 ]; then + git config --global --get-all $key | grep -Fq "${value}" && return 1 + git config --global --add "${key}" "${value}" + else + git config --get-all $key | grep -Fq "${value}" && return 1 + git config --add "${key}" "${value}" + fi +} + +register_aws() { + # Reusable regex patterns + local aws="(AWS|aws|Aws)?_?" quote="(\"|')" connect="\s*(:|=>|=)\s*" + local opt_quote="${quote}?" + add_config 'secrets.providers' 'git secrets --aws-provider' + add_config 'secrets.patterns' '[A-Z0-9]{20}' + add_config 'secrets.patterns' "${opt_quote}${aws}(SECRET|secret|Secret)?_?(ACCESS|access|Access)?_?(KEY|key|Key)${opt_quote}${connect}${opt_quote}[A-Za-z0-9/\+=]{40}${opt_quote}" + add_config 'secrets.patterns' "${opt_quote}${aws}(ACCOUNT|account|Account)_?(ID|id|Id)?${opt_quote}${connect}${opt_quote}[0-9]{4}\-?[0-9]{4}\-?[0-9]{4}${opt_quote}" + add_config 'secrets.allowed' 'AKIAIOSFODNN7EXAMPLE' + add_config 'secrets.allowed' "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" +} + +aws_provider() { + local fi="$1" + [ -z "$fi" ] && fi=~/.aws/credentials + # Find keys and ensure that special characters are escaped. + if [ -f $fi ]; then + awk -F "=" '/aws_access_key_id|aws_secret_access_key/ {print $2}' $fi \ + | tr -d ' "' \ + | sed 's/[]\.|$(){}?+*^]/\\&/g' + fi +} + +declare COMMAND="$1" FORCE=0 RECURSIVE=0 LITERAL=0 GLOBAL=0 ALLOWED=0 +# Shift off the command name +shift 1 +while [ "$#" -ne 0 ]; do + case "$1" in + -f) FORCE=1 ;; + -r) RECURSIVE=1 ;; + -a) ALLOWED=1 ;; + -l) LITERAL=1 ;; + --global) GLOBAL=1 ;; + --) shift; break ;; + esac + shift +done + +case "${COMMAND}" in + -h|--help|--) "$0" -h; exit 0 ;; + --add-provider) add_config "secrets.providers" "$@" ;; + --register-aws) register_aws ;; + --aws-provider) aws_provider "$1" ;; + --commit_msg_hook|--pre_commit_hook|--prepare_commit_msg_hook) + ${COMMAND:2} "$@" + ;; + --add) + if [ "${ALLOWED}" -eq 1 ]; then + add_config "secrets.allowed" "$1" + else + add_config "secrets.patterns" "$1" + fi + ;; + --scan) + if [ $# -eq 0 ]; then + scan_or_die "$(git ls-files)" + else + scan_or_die "$@" + fi + ;; + --list) + if [ "${GLOBAL}" -eq 1 ]; then + git config --global --get-regex secrets.* + else + git config --get-regex secrets.* + fi + ;; + --install) + DIRECTORY="$1" + if [ -z "${DIRECTORY}" ]; then + DIRECTORY=$(git rev-parse --git-dir) || die "Not in a Git repository" + elif [ -d "${DIRECTORY}"/.git ]; then + DIRECTORY="${DIRECTORY}/.git" + fi + mkdir -p "${DIRECTORY}/hooks" || die "Could not create dir: ${DIRECTORY}" + install_all_hooks "${DIRECTORY}" + ;; + *) echo "Unknown option: ${option}" && "$0" -h ;; +esac diff --git a/git-secrets.1 b/git-secrets.1 new file mode 100644 index 0000000..006d2cf --- /dev/null +++ b/git-secrets.1 @@ -0,0 +1,660 @@ +.\" Man page generated from reStructuredText. +. +.TH GIT-SECRETS "" "" "" +.SH NAME +git-secrets \- +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.sp +Prevents you from committing passwords and other sensitive information to a +git repository. +.SH SYNOPSIS +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-scan [\-r|\-\-recursive] [...] +git secrets \-\-install [\-f|\-\-force] [] +git secrets \-\-list [\-\-global] +git secrets \-\-add [\-a|\-\-allowed] [\-l|\-\-literal] [\-\-global] +git secrets \-\-add\-provider [\-\-global] [arguments...] +git secrets \-\-register\-aws [\-\-global] +git secrets \-\-aws\-provider [] +.ft P +.fi +.UNINDENT +.UNINDENT +.SH DESCRIPTION +.sp +\fBgit\-secrets\fP scans commits, commit messages, and \fB\-\-no\-ff\fP merges to +prevent adding secrets into your git repositories. If a commit, +commit message, or any commit in a \fB\-\-no\-ff\fP merge history matches one of +your configured prohibited regular expression patterns, then the commit is +rejected. +.SS Installing git\-secrets +.sp +\fBgit\-secrets\fP must be placed somewhere in your PATH so that it is picked up +by \fBgit\fP when running \fBgit secrets\fP\&. You can use \fBinstall\fP target of the +provided Makefile to install \fBgit secrets\fP and the man page. You can +customize the install path using the PREFIX and MANPREFIX variables. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +make install +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +\fBWARNING:\fP +.INDENT 0.0 +.INDENT 3.5 +You\(aqre not done yet! You MUST install the git hooks for every repo that +you wish to use with \fBgit secrets \-\-install\fP\&. +.UNINDENT +.UNINDENT +.SH OPTIONS +.SS Operation Modes +.sp +Each of these options must appear first on the command line. +.INDENT 0.0 +.TP +.B \fB\-\-install\fP +Installs hooks for a repository. Once the hooks are installed for a git +repository, commits and non\-ff merges for that repository will be prevented +from committing secrets. +.TP +.B \fB\-\-scan\fP +Scans one or more files for secrets. When a file contains a secret, the +matched text from the file being scanned will be written to stdout and the +script will exit with a non\-zero RC. Each matched line will be written with +the name of the file that matched, a colon, the line number that matched, +a colon, and then the line of text that matched. If no files are provided, +all files returned by \fBgit ls\-files\fP are scanned. +.TP +.B \fB\-\-list\fP +Lists the git\-secrets configuration for the current repo or in the global +git config. +.TP +.B \fB\-\-add\fP +Adds a prohibited or allowed pattern. +.TP +.B \fBadd\-provider\fP +Registers a secret provider. Secret providers are executables that when +invoked outputs prohibited patterns that \fBgit\-secrets\fP should treat as +prohibited. +.TP +.B \fB\-\-register\-aws\fP +Adds common AWS patterns to the git config and ensures that keys present +in \fB~/.aws/credentials\fP are not found in any commit. The following +checks are added: +.INDENT 7.0 +.IP \(bu 2 +AWS Access Key ID via \fB[A\-Z0\-9]{20}\fP +.IP \(bu 2 +AWS Secret Access Key assignments via ":" or "=" surrounded by optional +quotes +.IP \(bu 2 +AWS account ID assignments via ":" or "=" surrounded by optional quotes +.IP \(bu 2 +Allowed patterns for example AWS keys (\fBAKIAIOSFODNN7EXAMPLE\fP and +\fBwJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY\fP) +.IP \(bu 2 +Enables using \fB~/.aws/credentials\fP to scan for known credentials. You +can configure the path to a custom AWS credentials file by providing the +path to the file in the \fBsecrets.scan\-aws\-credentials\fP git config +setting. +.UNINDENT +.sp +\fBNOTE:\fP +.INDENT 7.0 +.INDENT 3.5 +While the patterns registered by this command should catch most +instances of AWS credentials, these patterns are \fBnot\fP guaranteed to +catch them \fBall\fP\&. \fBgit\-secrets\fP should be used as an extra means of +insurance \-\- you still need to do your due diligence to ensure that you +do not commit credentials to a repository. +.UNINDENT +.UNINDENT +.TP +.B \fB\-\-aws\-provider\fP +Secret provider that outputs credentials found in an INI file. You can +optionally provide the path to an ini file. +.UNINDENT +.SS Options for \fB\-\-install\fP +.INDENT 0.0 +.TP +.B \fB\-f, \-\-force\fP +Overwrites existing hooks if present. +.TP +.B \fB\fP +When provided, installs git hooks to the given directory. The current +directory is assumed if \fB\fP is not provided. +.sp +If the provided \fB\fP is not in a Git repository, the +directory will be created and hooks will be placed in +\fB/hooks\fP\&. This can be useful for creating Git template +directories using with \fBgit init \-\-template \fP\&. +.sp +You can run \fBgit init\fP on a repository that has already been initialized. +From the \fI\%git init documentation\fP: +.INDENT 7.0 +.INDENT 3.5 +From the git documentation: Running git init in an existing repository +is safe. It will not overwrite things that are already there. The +primary reason for rerunning git init is to pick up newly added +templates (or to move the repository to another place if +\fB\-\-separate\-git\-dir\fP is given). +.UNINDENT +.UNINDENT +.sp +The following git hooks are installed: +.INDENT 7.0 +.IP 1. 3 +\fBpre\-commit\fP: Used to check if any of the files changed in the commit +use prohibited patterns. +.IP 2. 3 +\fBcommit\-msg\fP: Used to determine if a commit message contains a +prohibited patterns. +.IP 3. 3 +\fBprepare\-commit\-msg\fP: Used to determine if a merge commit will +introduce a history that contains a prohibited pattern at any point. +Please note that this hook is only invoked for non fast\-forward merges. +.UNINDENT +.sp +\fBNOTE:\fP +.INDENT 7.0 +.INDENT 3.5 +Git only allows a single script to be executed per hook. If the +repository contains Debian style subdirectories like \fBpre\-commit.d\fP +and \fBcommit\-msg.d\fP, then the git hooks will be installed into these +directories, which assumes that you\(aqve configured the corresponding +hooks to execute all of the scripts found in these directories. If +these git subdirectories are not present, then the git hooks will be +installed to the git repo\(aqs \fB\&.git/hooks\fP directory. +.UNINDENT +.UNINDENT +.UNINDENT +.SS Examples +.sp +Install git hooks to the current directory: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +cd /path/to/my/repository +git secrets \-\-install +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Install git hooks to a repository other than the current directory: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-install \-/path/to/my/repository +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Create a git template that has \fBgit\-secrets\fP installed, and then copy that +template into a git repository: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-install ~/.git\-templates/git\-secrets +git init \-\-template ~/.git\-templates/git\-secrets +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Overwrite existing hooks if present: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-install \-f +.ft P +.fi +.UNINDENT +.UNINDENT +.SS Options for \fB\-\-scan\fP +.INDENT 0.0 +.TP +.B \fB\-r, \-\-recursive\fP +Scans the given files recursively. If a directory is encountered, the +directory will be scanned. If \fB\-r\fP is not provided, directories will be +ignored. +.TP +.B \fB...\fP +The path to one or more files on disk to scan for secrets. +.sp +If no files are provided, all files returned by \fBgit ls\-files\fP are +scanned. +.UNINDENT +.SS Examples +.sp +Scan all files in the repo: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-scan +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Scans a single file for secrets: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-scan /path/to/file +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Scans a directory recursively for secrets: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-scan \-r /path/to/directory +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Scans multiple files for secrets: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-scan /path/to/file /path/to/other/file +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +You can scan by globbing: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-scan /path/to/directory/* +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Scan from stdin: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +echo \(aqhello!\(aq | git secrets \-\-scan \- +.ft P +.fi +.UNINDENT +.UNINDENT +.SS Options for \fB\-\-list\fP +.INDENT 0.0 +.TP +.B \fB\-\-global\fP +Lists only git\-secrets configuration in the global git config. +.UNINDENT +.SS Options for \fB\-\-add\fP +.INDENT 0.0 +.TP +.B \fB\-\-global\fP +Adds patterns to the global git config +.TP +.B \fB\-l, \-\-literal\fP +Escapes special regular expression characters in the provided pattern so +that the pattern is searched for literally. +.TP +.B \fB\-a, \-\-allowed\fP +Mark the pattern as allowed instead of prohibited. Allowed patterns are +used to filter our false positives. +.TP +.B \fB\fP +The regex pattern to search. +.UNINDENT +.SS Examples +.sp +Adds a prohibited pattern to the current repo: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-add \(aq[A\-Z0\-9]{20}\(aq +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Adds a prohibited pattern to the global git config: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-add \-\-global \(aq[A\-Z0\-9]{20}\(aq +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Adds a string that is scanned for literally (\fB+\fP is escaped): +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-add \-\-literal \(aqfoo+bar\(aq +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Add an allowed pattern: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-add \-a \(aqallowed pattern\(aq +.ft P +.fi +.UNINDENT +.UNINDENT +.SS Options for \fB\-\-register\-aws\fP +.INDENT 0.0 +.TP +.B \fB\-\-global\fP +Adds AWS specific configuration variables to the global git config. +.UNINDENT +.SS Options for \fB\-\-aws\-provider\fP +.INDENT 0.0 +.TP +.B \fB[]\fP +If provided, specifies the custom path to an INI file to scan. If not +provided, \fB~/.aws/credentials\fP is assumed. +.UNINDENT +.SS Options for \fB\-\-add\-provider\fP +.INDENT 0.0 +.TP +.B \fB\-\-global\fP +Adds the provider to the global git config. +.TP +.B \fB\fP +Provider command to invoke. When invoked the command is expected to write +prohibited patterns separated by new lines to stdout. Any extra arguments +provided are passed on to the command. +.UNINDENT +.SS Examples +.sp +Registers a secret provider with arguments: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-add\-provider \-\- git secrets \-\-aws\-provider +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Cats secrets out of a file: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-add\-provider \-\- cat /path/to/secret/file/patterns +.ft P +.fi +.UNINDENT +.UNINDENT +.SH DEFINING PROHIBITED PATTERNS +.sp +egrep compatible regular expressions are used to determine if a commit or +commit message contains any prohibited patterns. These regular expressions are +defined using the \fBgit config\fP command. It is important to note that +different systems use different versions of egrep. For example, when running on +OS X, you will use a different version of egrep than when running on something +like Ubuntu (BSD vs GNU). +.sp +You can add prohibited regular expression patterns to your git config using +\fBgit secrets \-\-add \fP\&. +.SH IGNORING FALSE-POSITIVES +.sp +Sometimes a regular expression might match false positives. For example, git +commit SHAs look a lot like AWS access keys. You can specify many different +regular expression patterns as false positives using the following command: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-add \-\-allowed \(aqmy regex pattern\(aq +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +First, git\-secrets will extract all lines from a file that contain a prohibited +match. Included in the matched results will be the full path to the name of +the file that was matched, followed \(aq:\(aq, followed by the line number that was +matched, followed by the entire line from the file that was matched by a secret +pattern. Then, if you\(aqve defined allowed regular expressions, git\-secrets will +check to see if all of the matched lines match at least one of your registered +allowed regular expressions. If all of the lines that were flagged as secret +are canceled out by an allowed match, then the subject text does not contain +any secrets. If any of the matched lines are not matched by an allowed regular +expression, then git\-secrets will fail the commit/merge/message. +.sp +\fBIMPORTANT:\fP +.INDENT 0.0 +.INDENT 3.5 +Just as it is a bad practice to add prohibited patterns that are too +greedy, it is also a bad practice to add allowed patterns that are too +forgiving. Be sure to test out your patterns using ad\-hoc calls to +\fBgit secrets \-\-scan $filename\fP to ensure they are working as intended. +.UNINDENT +.UNINDENT +.SH SECRET PROVIDERS +.sp +Sometimes you want to check for an exact pattern match against a set of known +secrets. For example, you might want to ensure that no credentials present in +\fB~/.aws/credentials\fP ever show up in a commit. In these cases, it\(aqs better to +leave these secrets in one location rather than spread them out across git +repositories in git configs. You can use "secret providers" to fetch these +types of credentials. A secret provider is an executable that when invoked +outputs prohibited patterns separated by new lines. +.sp +You can add secret providers using the \fB\-\-add\-provider\fP command: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-add\-provider \-\- git secrets \-\-aws\-provider +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Notice the use of \fB\-\-\fP\&. This ensures that any arguments associated with the +provider are passed to the provider each time it is invoked when scanning +for secrets. +.SH EXAMPLE WALKTHROUGH +.sp +Let\(aqs take a look at an example. Given the following subject text (stored in +\fB/tmp/example\fP): +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +This is a test! +password=ex@mplepassword +password=****** +More test... +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +And the following registered patterns: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git config \-\-add \(aqpassword\es*=\es*.+\(aq +git config \-\-add \-\-allowed \-\-literal \(aqex@mplepassword\(aq +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Running \fBgit secrets \-\-scan /tmp/example\fP, the result will +result in the following error output: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +/tmp/example:3:password=****** + +[ERROR] Matched prohibited pattern + +Possible mitigations: +\- Mark false positives as allowed using: git config \-\-add secrets.allowed ... +\- List your configured patterns: git config \-\-get\-all secrets.patterns +\- List your configured allowed patterns: git config \-\-get\-all secrets.allowed +\- Use \-\-no\-verify if this is a one\-time false positive +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Breaking this down, the prohibited pattern value of \fBpassword\es*=\es*.+\fP will +match the following lines: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +/tmp/example:2:password=ex@mplepassword +/tmp/example:3:password=****** +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +\&...But the first match will be filtered out due to the fact that it matches the +allowed regular expression of \fBex@mplepassword\fP\&. Because there is still a +remaining line that did not match, it is considered a secret. +.sp +Because that matching lines are placed on lines that start with the filename +and line number (e.g., \fB/tmp/example:3:...\fP), you can create allowed +patterns that take filenames and line numbers into account in the regular +expression. For example, you could whitelist an entire file using something +like: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-add \-\-allowed \(aq/tmp/example:.*\(aq +git secrets \-\-scan /tmp/example && echo $? +# Outputs: 0 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Alternatively, you could whitelist a specific line number of a file if that +line is unlikely to change using something like the following: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-add \-\-allowed \(aq/tmp/example:3:.*\(aq +git secrets \-\-scan /tmp/example && echo $? +# Outputs: 0 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Keep this in mind when creating allowed patterns to ensure that your allowed +patterns are not inadvertantly matched due to the fact that the filename is +included in the subject text that allowed patterns are matched against. +.SH SKIPPING VALIDATION +.sp +Use the \fB\-\-no\-verify\fP option in the event of a false\-positive match in a +commit, merge, or commit message. This will skip the execution of the +git hook and allow you to make the commit or merge. +.SH ABOUT +.INDENT 0.0 +.IP \(bu 2 +Author: Michael Dowling <\fI\%https://github.com/mtdowling\fP> +.IP \(bu 2 +Issue tracker: This project\(aqs source code and issue tracker can be found at +\fI\%https://github.com/awslabs/git\-secrets\fP +.UNINDENT +.sp +Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +.\" Generated by docutils manpage writer. +. diff --git a/test/bats/LICENSE b/test/bats/LICENSE new file mode 100644 index 0000000..bac4eb2 --- /dev/null +++ b/test/bats/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2014 Sam Stephenson + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/test/bats/bin/bats b/test/bats/bin/bats new file mode 120000 index 0000000..a50a884 --- /dev/null +++ b/test/bats/bin/bats @@ -0,0 +1 @@ +../libexec/bats \ No newline at end of file diff --git a/test/bats/libexec/bats b/test/bats/libexec/bats new file mode 100755 index 0000000..71f392f --- /dev/null +++ b/test/bats/libexec/bats @@ -0,0 +1,142 @@ +#!/usr/bin/env bash +set -e + +version() { + echo "Bats 0.4.0" +} + +usage() { + version + echo "Usage: bats [-c] [-p | -t] [ ...]" +} + +help() { + usage + echo + echo " is the path to a Bats test file, or the path to a directory" + echo " containing Bats test files." + echo + echo " -c, --count Count the number of test cases without running any tests" + echo " -h, --help Display this help message" + echo " -p, --pretty Show results in pretty format (default for terminals)" + echo " -t, --tap Show results in TAP format" + echo " -v, --version Display the version number" + echo + echo " For more information, see https://github.com/sstephenson/bats" + echo +} + +resolve_link() { + $(type -p greadlink readlink | head -1) "$1" +} + +abs_dirname() { + local cwd="$(pwd)" + local path="$1" + + while [ -n "$path" ]; do + cd "${path%/*}" + local name="${path##*/}" + path="$(resolve_link "$name" || true)" + done + + pwd + cd "$cwd" +} + +expand_path() { + { cd "$(dirname "$1")" 2>/dev/null + local dirname="$PWD" + cd "$OLDPWD" + echo "$dirname/$(basename "$1")" + } || echo "$1" +} + +BATS_LIBEXEC="$(abs_dirname "$0")" +export BATS_PREFIX="$(abs_dirname "$BATS_LIBEXEC")" +export BATS_CWD="$(abs_dirname .)" +export PATH="$BATS_LIBEXEC:$PATH" + +options=() +arguments=() +for arg in "$@"; do + if [ "${arg:0:1}" = "-" ]; then + if [ "${arg:1:1}" = "-" ]; then + options[${#options[*]}]="${arg:2}" + else + index=1 + while option="${arg:$index:1}"; do + [ -n "$option" ] || break + options[${#options[*]}]="$option" + let index+=1 + done + fi + else + arguments[${#arguments[*]}]="$arg" + fi +done + +unset count_flag pretty +[ -t 0 ] && [ -t 1 ] && pretty="1" +[ -n "$CI" ] && pretty="" + +for option in "${options[@]}"; do + case "$option" in + "h" | "help" ) + help + exit 0 + ;; + "v" | "version" ) + version + exit 0 + ;; + "c" | "count" ) + count_flag="-c" + ;; + "t" | "tap" ) + pretty="" + ;; + "p" | "pretty" ) + pretty="1" + ;; + * ) + usage >&2 + exit 1 + ;; + esac +done + +if [ "${#arguments[@]}" -eq 0 ]; then + usage >&2 + exit 1 +fi + +filenames=() +for filename in "${arguments[@]}"; do + if [ -d "$filename" ]; then + shopt -s nullglob + for suite_filename in "$(expand_path "$filename")"/*.bats; do + filenames["${#filenames[@]}"]="$suite_filename" + done + shopt -u nullglob + else + filenames["${#filenames[@]}"]="$(expand_path "$filename")" + fi +done + +if [ "${#filenames[@]}" -eq 1 ]; then + command="bats-exec-test" +else + command="bats-exec-suite" +fi + +if [ -n "$pretty" ]; then + extended_syntax_flag="-x" + formatter="bats-format-tap-stream" +else + extended_syntax_flag="" + formatter="cat" +fi + +set -o pipefail execfail +exec "$command" $count_flag $extended_syntax_flag "${filenames[@]}" | "$formatter" diff --git a/test/bats/libexec/bats-exec-suite b/test/bats/libexec/bats-exec-suite new file mode 100755 index 0000000..29ab255 --- /dev/null +++ b/test/bats/libexec/bats-exec-suite @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -e + +count_only_flag="" +if [ "$1" = "-c" ]; then + count_only_flag=1 + shift +fi + +extended_syntax_flag="" +if [ "$1" = "-x" ]; then + extended_syntax_flag="-x" + shift +fi + +trap "kill 0; exit 1" int + +count=0 +for filename in "$@"; do + let count+="$(bats-exec-test -c "$filename")" +done + +if [ -n "$count_only_flag" ]; then + echo "$count" + exit +fi + +echo "1..$count" +status=0 +offset=0 +for filename in "$@"; do + index=0 + { + IFS= read -r # 1..n + while IFS= read -r line; do + case "$line" in + "begin "* ) + let index+=1 + echo "${line/ $index / $(($offset + $index)) }" + ;; + "ok "* | "not ok "* ) + [ -n "$extended_syntax_flag" ] || let index+=1 + echo "${line/ $index / $(($offset + $index)) }" + [ "${line:0:6}" != "not ok" ] || status=1 + ;; + * ) + echo "$line" + ;; + esac + done + } < <( bats-exec-test $extended_syntax_flag "$filename" ) + offset=$(($offset + $index)) +done + +exit "$status" diff --git a/test/bats/libexec/bats-exec-test b/test/bats/libexec/bats-exec-test new file mode 100755 index 0000000..8f3bd51 --- /dev/null +++ b/test/bats/libexec/bats-exec-test @@ -0,0 +1,346 @@ +#!/usr/bin/env bash +set -e +set -E +set -T + +BATS_COUNT_ONLY="" +if [ "$1" = "-c" ]; then + BATS_COUNT_ONLY=1 + shift +fi + +BATS_EXTENDED_SYNTAX="" +if [ "$1" = "-x" ]; then + BATS_EXTENDED_SYNTAX="$1" + shift +fi + +BATS_TEST_FILENAME="$1" +if [ -z "$BATS_TEST_FILENAME" ]; then + echo "usage: bats-exec " >&2 + exit 1 +elif [ ! -f "$BATS_TEST_FILENAME" ]; then + echo "bats: $BATS_TEST_FILENAME does not exist" >&2 + exit 1 +else + shift +fi + +BATS_TEST_DIRNAME="$(dirname "$BATS_TEST_FILENAME")" +BATS_TEST_NAMES=() + +load() { + local name="$1" + local filename + + if [ "${name:0:1}" = "/" ]; then + filename="${name}" + else + filename="$BATS_TEST_DIRNAME/${name}.bash" + fi + + [ -f "$filename" ] || { + echo "bats: $filename does not exist" >&2 + exit 1 + } + + source "${filename}" +} + +run() { + local e E T oldIFS + [[ ! "$-" =~ e ]] || e=1 + [[ ! "$-" =~ E ]] || E=1 + [[ ! "$-" =~ T ]] || T=1 + set +e + set +E + set +T + output="$("$@" 2>&1)" + status="$?" + oldIFS=$IFS + IFS=$'\n' lines=($output) + [ -z "$e" ] || set -e + [ -z "$E" ] || set -E + [ -z "$T" ] || set -T + IFS=$oldIFS +} + +setup() { + true +} + +teardown() { + true +} + +skip() { + BATS_TEST_SKIPPED=${1:-1} + BATS_TEST_COMPLETED=1 + exit 0 +} + +bats_test_begin() { + BATS_TEST_DESCRIPTION="$1" + if [ -n "$BATS_EXTENDED_SYNTAX" ]; then + echo "begin $BATS_TEST_NUMBER $BATS_TEST_DESCRIPTION" >&3 + fi + setup +} + +bats_test_function() { + local test_name="$1" + BATS_TEST_NAMES["${#BATS_TEST_NAMES[@]}"]="$test_name" +} + +bats_capture_stack_trace() { + BATS_PREVIOUS_STACK_TRACE=( "${BATS_CURRENT_STACK_TRACE[@]}" ) + BATS_CURRENT_STACK_TRACE=() + + local test_pattern=" $BATS_TEST_NAME $BATS_TEST_SOURCE" + local setup_pattern=" setup $BATS_TEST_SOURCE" + local teardown_pattern=" teardown $BATS_TEST_SOURCE" + + local frame + local index=1 + + while frame="$(caller "$index")"; do + BATS_CURRENT_STACK_TRACE["${#BATS_CURRENT_STACK_TRACE[@]}"]="$frame" + if [[ "$frame" = *"$test_pattern" || \ + "$frame" = *"$setup_pattern" || \ + "$frame" = *"$teardown_pattern" ]]; then + break + else + let index+=1 + fi + done + + BATS_SOURCE="$(bats_frame_filename "${BATS_CURRENT_STACK_TRACE[0]}")" + BATS_LINENO="$(bats_frame_lineno "${BATS_CURRENT_STACK_TRACE[0]}")" +} + +bats_print_stack_trace() { + local frame + local index=1 + local count="${#@}" + + for frame in "$@"; do + local filename="$(bats_trim_filename "$(bats_frame_filename "$frame")")" + local lineno="$(bats_frame_lineno "$frame")" + + if [ $index -eq 1 ]; then + echo -n "# (" + else + echo -n "# " + fi + + local fn="$(bats_frame_function "$frame")" + if [ "$fn" != "$BATS_TEST_NAME" ]; then + echo -n "from function \`$fn' " + fi + + if [ $index -eq $count ]; then + echo "in test file $filename, line $lineno)" + else + echo "in file $filename, line $lineno," + fi + + let index+=1 + done +} + +bats_print_failed_command() { + local frame="$1" + local status="$2" + local filename="$(bats_frame_filename "$frame")" + local lineno="$(bats_frame_lineno "$frame")" + + local failed_line="$(bats_extract_line "$filename" "$lineno")" + local failed_command="$(bats_strip_string "$failed_line")" + echo -n "# \`${failed_command}' " + + if [ $status -eq 1 ]; then + echo "failed" + else + echo "failed with status $status" + fi +} + +bats_frame_lineno() { + local frame="$1" + local lineno="${frame%% *}" + echo "$lineno" +} + +bats_frame_function() { + local frame="$1" + local rest="${frame#* }" + local fn="${rest%% *}" + echo "$fn" +} + +bats_frame_filename() { + local frame="$1" + local rest="${frame#* }" + local filename="${rest#* }" + + if [ "$filename" = "$BATS_TEST_SOURCE" ]; then + echo "$BATS_TEST_FILENAME" + else + echo "$filename" + fi +} + +bats_extract_line() { + local filename="$1" + local lineno="$2" + sed -n "${lineno}p" "$filename" +} + +bats_strip_string() { + local string="$1" + printf "%s" "$string" | sed -e "s/^[ "$'\t'"]*//" -e "s/[ "$'\t'"]*$//" +} + +bats_trim_filename() { + local filename="$1" + local length="${#BATS_CWD}" + + if [ "${filename:0:length+1}" = "${BATS_CWD}/" ]; then + echo "${filename:length+1}" + else + echo "$filename" + fi +} + +bats_debug_trap() { + if [ "$BASH_SOURCE" != "$1" ]; then + bats_capture_stack_trace + fi +} + +bats_error_trap() { + BATS_ERROR_STATUS="$?" + BATS_ERROR_STACK_TRACE=( "${BATS_PREVIOUS_STACK_TRACE[@]}" ) + trap - debug +} + +bats_teardown_trap() { + trap "bats_exit_trap" exit + local status=0 + teardown >>"$BATS_OUT" 2>&1 || status="$?" + + if [ $status -eq 0 ]; then + BATS_TEARDOWN_COMPLETED=1 + elif [ -n "$BATS_TEST_COMPLETED" ]; then + BATS_ERROR_STATUS="$status" + BATS_ERROR_STACK_TRACE=( "${BATS_CURRENT_STACK_TRACE[@]}" ) + fi + + bats_exit_trap +} + +bats_exit_trap() { + local status + local skipped + trap - err exit + + skipped="" + if [ -n "$BATS_TEST_SKIPPED" ]; then + skipped=" # skip" + if [ "1" != "$BATS_TEST_SKIPPED" ]; then + skipped+=" ($BATS_TEST_SKIPPED)" + fi + fi + + if [ -z "$BATS_TEST_COMPLETED" ] || [ -z "$BATS_TEARDOWN_COMPLETED" ]; then + echo "not ok $BATS_TEST_NUMBER $BATS_TEST_DESCRIPTION" >&3 + bats_print_stack_trace "${BATS_ERROR_STACK_TRACE[@]}" >&3 + bats_print_failed_command "${BATS_ERROR_STACK_TRACE[${#BATS_ERROR_STACK_TRACE[@]}-1]}" "$BATS_ERROR_STATUS" >&3 + sed -e "s/^/# /" < "$BATS_OUT" >&3 + status=1 + else + echo "ok ${BATS_TEST_NUMBER}${skipped} ${BATS_TEST_DESCRIPTION}" >&3 + status=0 + fi + + rm -f "$BATS_OUT" + exit "$status" +} + +bats_perform_tests() { + echo "1..$#" + test_number=1 + status=0 + for test_name in "$@"; do + "$0" $BATS_EXTENDED_SYNTAX "$BATS_TEST_FILENAME" "$test_name" "$test_number" || status=1 + let test_number+=1 + done + exit "$status" +} + +bats_perform_test() { + BATS_TEST_NAME="$1" + if [ "$(type -t "$BATS_TEST_NAME" || true)" = "function" ]; then + BATS_TEST_NUMBER="$2" + if [ -z "$BATS_TEST_NUMBER" ]; then + echo "1..1" + BATS_TEST_NUMBER="1" + fi + + BATS_TEST_COMPLETED="" + BATS_TEARDOWN_COMPLETED="" + trap "bats_debug_trap \"\$BASH_SOURCE\"" debug + trap "bats_error_trap" err + trap "bats_teardown_trap" exit + "$BATS_TEST_NAME" >>"$BATS_OUT" 2>&1 + BATS_TEST_COMPLETED=1 + + else + echo "bats: unknown test name \`$BATS_TEST_NAME'" >&2 + exit 1 + fi +} + +if [ -z "$TMPDIR" ]; then + BATS_TMPDIR="/tmp" +else + BATS_TMPDIR="${TMPDIR%/}" +fi + +BATS_TMPNAME="$BATS_TMPDIR/bats.$$" +BATS_PARENT_TMPNAME="$BATS_TMPDIR/bats.$PPID" +BATS_OUT="${BATS_TMPNAME}.out" + +bats_preprocess_source() { + BATS_TEST_SOURCE="${BATS_TMPNAME}.src" + { tr -d '\r' < "$BATS_TEST_FILENAME"; echo; } | bats-preprocess > "$BATS_TEST_SOURCE" + trap "bats_cleanup_preprocessed_source" err exit + trap "bats_cleanup_preprocessed_source; exit 1" int +} + +bats_cleanup_preprocessed_source() { + rm -f "$BATS_TEST_SOURCE" +} + +bats_evaluate_preprocessed_source() { + if [ -z "$BATS_TEST_SOURCE" ]; then + BATS_TEST_SOURCE="${BATS_PARENT_TMPNAME}.src" + fi + source "$BATS_TEST_SOURCE" +} + +exec 3<&1 + +if [ "$#" -eq 0 ]; then + bats_preprocess_source + bats_evaluate_preprocessed_source + + if [ -n "$BATS_COUNT_ONLY" ]; then + echo "${#BATS_TEST_NAMES[@]}" + else + bats_perform_tests "${BATS_TEST_NAMES[@]}" + fi +else + bats_evaluate_preprocessed_source + bats_perform_test "$@" +fi diff --git a/test/bats/libexec/bats-format-tap-stream b/test/bats/libexec/bats-format-tap-stream new file mode 100755 index 0000000..614768f --- /dev/null +++ b/test/bats/libexec/bats-format-tap-stream @@ -0,0 +1,165 @@ +#!/usr/bin/env bash +set -e + +# Just stream the TAP output (sans extended syntax) if tput is missing +command -v tput >/dev/null || exec grep -v "^begin " + +header_pattern='[0-9]+\.\.[0-9]+' +IFS= read -r header + +if [[ "$header" =~ $header_pattern ]]; then + count="${header:3}" + index=0 + failures=0 + skipped=0 + name="" + count_column_width=$(( ${#count} * 2 + 2 )) +else + # If the first line isn't a TAP plan, print it and pass the rest through + printf "%s\n" "$header" + exec cat +fi + +update_screen_width() { + screen_width="$(tput cols)" + count_column_left=$(( $screen_width - $count_column_width )) +} + +trap update_screen_width WINCH +update_screen_width + +begin() { + go_to_column 0 + printf_with_truncation $(( $count_column_left - 1 )) " %s" "$name" + clear_to_end_of_line + go_to_column $count_column_left + printf "%${#count}s/${count}" "$index" + go_to_column 1 +} + +pass() { + go_to_column 0 + printf " ✓ %s" "$name" + advance +} + +skip() { + local reason="$1" + [ -z "$reason" ] || reason=": $reason" + go_to_column 0 + printf " - %s (skipped%s)" "$name" "$reason" + advance +} + +fail() { + go_to_column 0 + set_color 1 bold + printf " ✗ %s" "$name" + advance +} + +log() { + set_color 1 + printf " %s\n" "$1" + clear_color +} + +summary() { + printf "\n%d test%s" "$count" "$(plural "$count")" + + printf ", %d failure%s" "$failures" "$(plural "$failures")" + + if [ "$skipped" -gt 0 ]; then + printf ", %d skipped" "$skipped" + fi + + printf "\n" +} + +printf_with_truncation() { + local width="$1" + shift + local string="$(printf "$@")" + + if [ "${#string}" -gt "$width" ]; then + printf "%s..." "${string:0:$(( $width - 4 ))}" + else + printf "%s" "$string" + fi +} + +go_to_column() { + local column="$1" + printf "\x1B[%dG" $(( $column + 1 )) +} + +clear_to_end_of_line() { + printf "\x1B[K" +} + +advance() { + clear_to_end_of_line + echo + clear_color +} + +set_color() { + local color="$1" + local weight="$2" + printf "\x1B[%d;%dm" $(( 30 + $color )) "$( [ "$weight" = "bold" ] && echo 1 || echo 22 )" +} + +clear_color() { + printf "\x1B[0m" +} + +plural() { + [ "$1" -eq 1 ] || echo "s" +} + +_buffer="" + +buffer() { + _buffer="${_buffer}$("$@")" +} + +flush() { + printf "%s" "$_buffer" + _buffer="" +} + +finish() { + flush + printf "\n" +} + +trap finish EXIT + +while IFS= read -r line; do + case "$line" in + "begin "* ) + let index+=1 + name="${line#* $index }" + buffer begin + flush + ;; + "ok "* ) + skip_expr="ok $index # skip (\(([^)]*)\))?" + if [[ "$line" =~ $skip_expr ]]; then + let skipped+=1 + buffer skip "${BASH_REMATCH[2]}" + else + buffer pass + fi + ;; + "not ok "* ) + let failures+=1 + buffer fail + ;; + "# "* ) + buffer log "${line:2}" + ;; + esac +done + +buffer summary diff --git a/test/bats/libexec/bats-preprocess b/test/bats/libexec/bats-preprocess new file mode 100755 index 0000000..04297ed --- /dev/null +++ b/test/bats/libexec/bats-preprocess @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -e + +encode_name() { + local name="$1" + local result="test_" + + if [[ ! "$name" =~ [^[:alnum:]\ _-] ]]; then + name="${name//_/-5f}" + name="${name//-/-2d}" + name="${name// /_}" + result+="$name" + else + local length="${#name}" + local char i + + for ((i=0; i "$BATS_TMPDIR/test.txt" + repo_run git-secrets --scan "$BATS_TMPDIR/test.txt" + [ $status -eq 0 ] +} + +@test "Scans all files when no file provided" { + setup_bad_repo + repo_run git-secrets --scan + [ $status -eq 1 ] +} + +@test "Scans recursively" { + setup_bad_repo + mkdir -p $TEST_REPO/foo/bar/baz + echo '@todo more stuff' > $TEST_REPO/foo/bar/baz/data.txt + repo_run git-secrets --scan -r $TEST_REPO/foo + [ $status -eq 1 ] +} + +@test "Scans recursively only if -r is given" { + setup_bad_repo + mkdir -p $TEST_REPO/foo/bar/baz + echo '@todo more stuff' > $TEST_REPO/foo/bar/baz/data.txt + repo_run git-secrets --scan $TEST_REPO/foo + [ $status -eq 0 ] +} + +@test "Excludes allowed patterns from failures" { + git config --add secrets.patterns 'foo="baz{1,5}"' + git config --add secrets.allowed 'foo="bazzz"' + echo 'foo="bazzz" is ok because 3 "z"s' > "$BATS_TMPDIR/test.txt" + repo_run git-secrets --scan "$BATS_TMPDIR/test.txt" + [ $status -eq 0 ] + echo 'This is NOT: ok foo="bazzzz"' > "$BATS_TMPDIR/test.txt" + repo_run git-secrets --scan "$BATS_TMPDIR/test.txt" + [ $status -eq 1 ] +} + +@test "Prohibited matches exits 1" { + file="$TEST_REPO/test.txt" + echo '@todo stuff' > $file + echo 'this is forbidden right?' >> $file + repo_run git-secrets --scan $file + [ $status -eq 1 ] + [ "${lines[0]}" == "$file:1:@todo stuff" ] + [ "${lines[1]}" == "$file:2:this is forbidden right?" ] +} + +@test "Only matches on word boundaries" { + file="$TEST_REPO/test.txt" + # Note that the following does not match as it is not a word. + echo 'mesa Jar Jar Binks' > $file + # The following do match because they are in word boundaries. + echo 'foo.me' >> $file + echo '"me"' >> $file + repo_run git-secrets --scan $file + [ $status -eq 1 ] + [ "${lines[0]}" == "$file:2:foo.me" ] + [ "${lines[1]}" == "$file:3:\"me\"" ] +} + +@test "Can scan from stdin using -" { + echo "foo" | "${BATS_TEST_DIRNAME}/../git-secrets" --scan - + echo "me" | "${BATS_TEST_DIRNAME}/../git-secrets" --scan - && exit 1 || true +} + +@test "installs hooks for repo" { + setup_bad_repo + repo_run git-secrets --install $TEST_REPO + [ -f $TEST_REPO/.git/hooks/pre-commit ] + [ -f $TEST_REPO/.git/hooks/prepare-commit-msg ] + [ -f $TEST_REPO/.git/hooks/commit-msg ] +} + +@test "fails if hook exists and no -f" { + repo_run git-secrets --install $TEST_REPO + repo_run git-secrets --install $TEST_REPO + [ $status -eq 1 ] +} + +@test "Overwrites hooks if -f is given" { + repo_run git-secrets --install $TEST_REPO + repo_run git-secrets --install -f $TEST_REPO + [ $status -eq 0 ] +} + +@test "installs hooks for repo with Debian style directories" { + setup_bad_repo + mkdir $TEST_REPO/.git/hooks/pre-commit.d + mkdir $TEST_REPO/.git/hooks/prepare-commit-msg.d + mkdir $TEST_REPO/.git/hooks/commit-msg.d + run git-secrets --install $TEST_REPO + [ -f $TEST_REPO/.git/hooks/pre-commit.d/git-secrets ] + [ -f $TEST_REPO/.git/hooks/prepare-commit-msg.d/git-secrets ] + [ -f $TEST_REPO/.git/hooks/commit-msg.d/git-secrets ] +} + +@test "installs hooks to template directory" { + setup_bad_repo + run git-secrets --install $TEMPLATE_DIR + [ $status -eq 0 ] + run git init --template $TEMPLATE_DIR + [ $status -eq 0 ] + [ -f "${TEST_REPO}/.git/hooks/pre-commit" ] + [ -f "${TEST_REPO}/.git/hooks/prepare-commit-msg" ] + [ -f "${TEST_REPO}/.git/hooks/commit-msg" ] +} + +@test "Scans using keys from credentials file" { + echo 'aws_access_key_id = abc123' > $BATS_TMPDIR/test.ini + echo 'aws_secret_access_key=foobaz' >> $BATS_TMPDIR/test.ini + echo 'aws_access_key_id = "Bernard"' >> $BATS_TMPDIR/test.ini + echo 'aws_secret_access_key= "Laverne"' >> $BATS_TMPDIR/test.ini + echo 'aws_access_key_id= Hoagie+man' >> $BATS_TMPDIR/test.ini + cd $TEST_REPO + run git secrets --aws-provider $BATS_TMPDIR/test.ini + [ $status -eq 0 ] + echo "$output" | grep -F "foobaz" + echo "$output" | grep -F "abc123" + echo "$output" | grep -F "Bernard" + echo "$output" | grep -F "Laverne" + echo "$output" | grep -F 'Hoagie\+man' + run git secrets --add-provider -- git secrets --aws-provider $BATS_TMPDIR/test.ini + [ $status -eq 0 ] + echo '(foobaz) test' > $TEST_REPO/bad_file + echo "abc123 test" >> $TEST_REPO/bad_file + echo 'Bernard test' >> $TEST_REPO/bad_file + echo 'Laverne test' >> $TEST_REPO/bad_file + echo 'Hoagie+man test' >> $TEST_REPO/bad_file + repo_run git-secrets --scan $TEST_REPO/bad_file + [ $status -eq 1 ] + echo "$output" | grep "foobaz" + echo "$output" | grep "abc123" + echo "$output" | grep "Bernard" + echo "$output" | grep "Laverne" + echo "$output" | grep -F 'Hoagie+man' +} + +@test "Lists secrets for a repo" { + repo_run git-secrets --list + [ $status -eq 0 ] + echo "$output" | grep -F 'secrets.patterns @todo' + echo "$output" | grep -F 'secrets.patterns forbidden|me' +} + +@test "Adds secrets to a repo and de-dedupes" { + repo_run git-secrets --add 'testing+123' + [ $status -eq 0 ] + repo_run git-secrets --add 'testing+123' + [ $status -eq 1 ] + repo_run git-secrets --add --literal 'testing+abc' + [ $status -eq 0 ] + repo_run git-secrets --add -l 'testing+abc' + [ $status -eq 1 ] + repo_run git-secrets --list + echo "$output" | grep -F 'secrets.patterns @todo' + echo "$output" | grep -F 'secrets.patterns forbidden|me' + echo "$output" | grep -F 'secrets.patterns testing+123' + echo "$output" | grep -F 'secrets.patterns testing\+abc' +} + +@test "Adds allowed patterns to a repo and de-dedupes" { + repo_run git-secrets --add -a 'testing+123' + [ $status -eq 0 ] + repo_run git-secrets --add --allowed 'testing+123' + [ $status -eq 1 ] + repo_run git-secrets --add -a -l 'testing+abc' + [ $status -eq 0 ] + repo_run git-secrets --add -a -l 'testing+abc' + [ $status -eq 1 ] + repo_run git-secrets --list + echo "$output" | grep -F 'secrets.patterns @todo' + echo "$output" | grep -F 'secrets.patterns forbidden|me' + echo "$output" | grep -F 'secrets.allowed testing+123' + echo "$output" | grep -F 'secrets.allowed testing\+abc' +} + +@test "Adds common AWS patterns" { + repo_run git config --unset-all secrets + repo_run git-secrets --register-aws + git config --local --get secrets.providers + repo_run git-secrets --list + echo "$output" | grep -F '[A-Z0-9]{20}' + echo "$output" | grep "AKIAIOSFODNN7EXAMPLE" + echo "$output" | grep "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" +} + +@test "Adds providers" { + repo_run git-secrets --add-provider -- echo foo baz bar + [ $status -eq 0 ] + repo_run git-secrets --add-provider -- echo bam + [ $status -eq 0 ] + repo_run git-secrets --list + echo "$output" | grep -F 'echo foo baz bar' + echo "$output" | grep -F 'echo bam' + echo 'foo baz bar' > $TEST_REPO/bad_file + echo 'bam' >> $TEST_REPO/bad_file + repo_run git-secrets --scan $TEST_REPO/bad_file + [ $status -eq 1 ] + echo "$output" | grep -F 'foo baz bar' + echo "$output" | grep -F 'bam' +} diff --git a/test/pre-commit.bats b/test/pre-commit.bats new file mode 100644 index 0000000..898bcf4 --- /dev/null +++ b/test/pre-commit.bats @@ -0,0 +1,27 @@ +#!/usr/bin/env bats +load test_helper + +@test "Rejects commits with prohibited patterns in changeset" { + setup_bad_repo + repo_run git-secrets --install $TEST_REPO + cd $TEST_REPO + run git commit -m 'Contents are bad not the message' + [ $status -eq 1 ] + [ "${lines[0]}" == "data.txt:1:@todo more stuff" ] + [ "${lines[1]}" == "failure1.txt:1:another line... forbidden" ] + [ "${lines[2]}" == "failure2.txt:1:me" ] +} + +@test "Allows commits that do not match prohibited patterns" { + setup_good_repo + repo_run git-secrets --install $TEST_REPO + cd $TEST_REPO + run git commit -m 'This is fine' + [ $status -eq 0 ] + # Ensure deleted files are filtered out of the grep + rm $TEST_REPO/data.txt + echo 'aaa' $TEST_REPO/data_2.txt + run git add -A + run git commit -m 'This is also fine' + [ $status -eq 0 ] +} diff --git a/test/prepare-commit-msg.bats b/test/prepare-commit-msg.bats new file mode 100644 index 0000000..a211c13 --- /dev/null +++ b/test/prepare-commit-msg.bats @@ -0,0 +1,33 @@ +#!/usr/bin/env bats +load test_helper + +@test "Rejects merges with prohibited patterns in history" { + setup_good_repo + repo_run git-secrets --install $TEST_REPO + cd $TEST_REPO + git commit -m 'OK' + git checkout -b feature + echo '@todo' > data.txt + git add -A + git commit -m 'Bad commit' --no-verify + echo 'Fixing!' > data.txt + git add -A + git commit -m 'Fixing commit' + git checkout master + run git merge --no-ff feature + [ $status -eq 1 ] +} + +@test "Allows merges that do not match prohibited patterns" { + setup_good_repo + cd $TEST_REPO + repo_run git-secrets --install + git commit -m 'OK' + git checkout -b feature + echo 'Not bad' > data.txt + git add -A + git commit -m 'Good commit' + git checkout master + run git merge --no-ff feature + [ $status -eq 0 ] +} diff --git a/test/test_helper.bash b/test/test_helper.bash new file mode 100644 index 0000000..376e58f --- /dev/null +++ b/test/test_helper.bash @@ -0,0 +1,57 @@ +#!/bin/bash +export TEST_REPO="$BATS_TMPDIR/test-repo" +export TEMPLATE_DIR="${BATS_TMPDIR}/template" +INITIAL_PATH="${PATH}" + +setup() { + setup_repo + [ -d "${TEMPLATE_DIR}" ] && rm -rf "${TEMPLATE_DIR}" + export PATH="${BATS_TEST_DIRNAME}/..:${INITIAL_PATH}" + cd $TEST_REPO +} + +teardown() { + delete_repo + export PATH="${INITIAL_PATH}" +} + +delete_repo() { + [ -d $TEST_REPO ] && rm -rf $TEST_REPO || true +} + +setup_repo() { + delete_repo + mkdir -p $TEST_REPO + cd $TEST_REPO + git init + git config --local --add secrets.patterns '@todo' + git config --local --add secrets.patterns 'forbidden|me' + cd - +} + +repo_run() { + cmd="$1" + shift + cd "${TEST_REPO}" + run "${BATS_TEST_DIRNAME}/../${cmd}" $@ + cd - +} + +# Creates a repo that should fail +setup_bad_repo() { + cd $TEST_REPO + echo '@todo more stuff' > $TEST_REPO/data.txt + echo 'hi there' > $TEST_REPO/ok.txt + echo 'another line... forbidden' > $TEST_REPO/failure1.txt + echo 'me' > $TEST_REPO/failure2.txt + git add -A + cd - +} + +# Creates a repo that does not fail +setup_good_repo() { + cd $TEST_REPO + echo 'hello!' > $TEST_REPO/data.txt + git add -A + cd - +}