Version 2.0.0

.github/workflows/build.yml

@@ -1,31 +1,35 @@
-name: build
+name: main
 
 on:
   push:
-    branches: [ master ]
+    branches:
+      - master
+
   pull_request:
-    branches: [ master ]
+    branches:
+      - master
 
 jobs:
-  build:
-
+  main:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: [3.6, 3.7, 3.8, 3.9]
 
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        make install-dev
-    - name: Lint
-      run: |
-        make lint
-    - name: Test and coverage
-      run: |
-        make coverage
+      - name: checkout
+        uses: actions/checkout@v2
+
+      - name: setup python${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: install
+        run: make install-dev
+
+      - name: lint
+        run: make lint
+
+      - name: coverage
+        run: make coverage

.github/workflows/publish.yml

@@ -2,29 +2,36 @@ name: publish
 
 on:
   push:
-    branches: [ master ]
+    branches:
+      - master
 
 jobs:
   deploy:
-
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.9]
 
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: '3.x'
-    - name: Install dependencies
-      run: |
-        make install-dev
-    - name: Test
-      run: |
-        make test
-    - name: Publish
-      env:
-        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-      run: |
-        python setup.py sdist bdist_wheel
-        twine upload --skip-existing dist/*
+      - name: checkout
+        uses: actions/checkout@v2
+
+      - name: setup python${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: install
+        run: make install-dev
+
+      - name: test
+        run: make test
+
+      - name: coverage
+        run: make coverage
+
+      - name: publish
+        run: make publish
+        env:
+          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}

.gitignore

@@ -75,6 +75,7 @@ share/python-wheels/
 *.egg
 MANIFEST
 dev
+pip.conf
 
 # PyInstaller
 #  Usually these files are written by a python script from a template

Makefile

@@ -8,15 +8,17 @@ flake8-lint:
 	flake8 whispers/ tests/
 
 isort-lint:
-	isort --check-only --recursive whispers/ tests/
+	isort --check-only whispers/ tests/
 
 black-lint:
 	black --check whispers/ tests/
 
 lint: isort-lint black-lint flake8-lint
 
 format:
-	isort --recursive whispers/ tests/
+	autoflake --in-place --recursive --remove-all-unused-imports whispers/ tests/
+	autopep8 --in-place --recursive --aggressive --aggressive whispers/ tests/
+	isort whispers/ tests/
 	black whispers/ tests/
 
 unit:
@@ -35,17 +37,13 @@ docker:
 	docker rmi -f $$(docker images --filter "dangling=true" -q --no-trunc)
 
 freeze:
-	CUSTOM_COMPILE_COMMAND="make freeze" pip-compile --no-index --output-file requirements.txt setup.py
+	CUSTOM_COMPILE_COMMAND="make freeze" pip-compile --no-emit-index-url --output-file=- > requirements.txt
 
-freeze-upgrade:
-	CUSTOM_COMPILE_COMMAND="make freeze-upgrade" pip-compile --no-index --upgrade --output-file requirements.txt setup.py
-
-dist:
-	rm -rf dist
-	python3 setup.py sdist
-	python3 -m twine upload dist/*
+publish:
+	python3 setup.py sdist bdist_wheel
+	twine upload --skip-existing dist/*
 
 test-pip:
 	python3 -m pip install --index-url https://test.pypi.org/simple/ --no-deps whispers
 
-.PHONY: install install-dev isort-lint black-lint flake8-lint format lint unit coverage test dist
+.PHONY: install install-dev isort-lint black-lint flake8-lint format lint unit coverage test publish

README.md

@@ -36,27 +36,18 @@ The following commonly used formats are currently supported:
 * pip.conf
 * conf / ini
 * Dockerfile
-* Dockercfg
 * Shell scripts
 * Python3
 
 Python3 files are parsed as ASTs because of native language support.
 
-## Declaration & Assignment Formats
-The following language files are parsed as text, and checked for common variable declaration and assignment patterns:
-* JavaScript
-* Java
-* Go
-* PHP
-
 ## Special Formats
 * AWS credentials files
 * JDBC connection strings
 * Jenkins config files
 * SpringFramework Beans config files
 * Java Properties files
 * Dockercfg private registry auth files
-* Github tokens
 
 ## Installation
 
@@ -76,24 +67,30 @@ make install
 ### CLI
 ```
 whispers --help
+whispers --version
 whispers --info
-whispers source/code/fileOrDir
-whispers --config config.yml source/code/fileOrDir
-whispers --output /tmp/secrets.yml source/code/fileOrDir
-whispers --rules aws-id,aws-secret source/code/fileOrDir
-whispers --severity BLOCKER,CRITICAL source/code/fileOrDir
-whispers --exitcode 7 source/code/fileOrDir
+whispers --config config.yml target/file/or/dir
+whispers --output /tmp/secrets.out target/file/or/dir
+whispers --rules aws-id,aws-secret target/file/or/dir
+whispers --severity BLOCKER,CRITICAL target/file/or/dir
+whispers --exitcode 7 target/file/or/dir
+whispers target/file/or/dir
 ```
+
 ### Python
 ```python
-from whispers.cli import parse_args
-from whispers.core import run
+from whispers.core.args import parse_args
+from whispers.main import run
+
+args = parse_args([
+  "-c", "whispers/config.yml",
+  "-r", "apikey,aws-secret,password",
+  "-s", "BLOCKER,CRITICAL,MAJOR",
+  "tests/fixtures"
+])
 
-src = "tests/fixtures"
-configfile = "whispers/config.yml"
-args = parse_args(["-c", configfile, src])
 for secret in run(args):
-  print(secret)
+  print(f"[{secret.file}:{secret.line}] {secret.key} = {secret.value}")
 ```
 
 ## Config
@@ -103,29 +100,35 @@ There are several configuration options available in Whispers. It’s possible t
 ```yaml
 include:
   files:
-    - "**/*.yml"
+    - "**/*.yml"  # glob
 
 exclude:
   files:
-    - "**/test/**/*"
-    - "**/tests/**/*"
+    - .*/tests?/  # regex
   keys:
-    - ^foo
+    - ^foo        # regex
   values:
-    - bar$
+    - bar$        # regex
 
 rules:
-  starks:
+  - password
+  - privatekey
+  - id: starks
     message: Whispers from the North
     severity: CRITICAL
     value:
       regex: (Aria|Ned) Stark
       ignorecase: True
+
+severity:
+  - CRITICAL
+  - BLOCKER
+  - MAJOR
 ```
 
 The fastest way to tweak detection (ie: remove false positives and unwanted results) is to copy the default [config.yml](whispers/config.yml) into a new file, adapt it, and pass it as an argument to Whispers.
 
-`whispers --config config.yml --rules starks src/file/or/dir`
+For example: `whispers -c config.yml -r starks target`
 
 
 ## Custom Rules
@@ -135,33 +138,40 @@ Rules specify the actual things that should be pulled out from key-value pairs.
 - Custom rules can be added to [whispers/rules](whispers/rules/)
 
 ```yaml
-rule-id:  # unique rule name
+- id: rule-id                 # unique rule name
   description: Values formatted like AWS Session Token
   message: AWS Session Token  # report will show this message
   severity: BLOCKER           # one of BLOCKER, CRITICAL, MAJOR, MINOR, INFO
 
-  key:        # specify key format
+  key:                        # specify key format
     regex: (aws.?session.?token)?
-    ignorecase: True   # case-insensitive matching
+    ignorecase: True          # case-insensitive matching
 
-  value:      # specify value format
+  value:                      # specify value format
     regex: ^(?=.*[a-z])(?=.*[A-Z])[A-Za-z0-9\+\/]{270,450}$
-    ignorecase: False  # case-sensitive matching
-    minlen: 270        # value is at least this long
-    isBase64: True     # value is base64-encoded
-    isAscii: False     # value is binary data when decoded
-    isUri: False       # value is not formatted like a URI
-
-  similar: 0.35        # maximum allowed similarity between key and value 
-                       # (1.0 being exactly the same)
+    ignorecase: False         # case-sensitive matching
+    minlen: 270               # value is at least this long
+    isBase64: True            # value is base64-encoded
+    isAscii: False            # value is binary data when decoded
+    isUri: False              # value is not formatted like a URI
+
+  similar: 0.35               # maximum allowed similarity between key and value 
+                              # (1.0 being exactly the same)
 ```
 
 
 ## Plugins
-All parsing functionality is implemented via plugins. Each plugin implements a class with the `pairs()` method that runs through files and returns the key-value pairs to be checked with rules. 
+All parsing functionality is implemented via plugins. Each plugin implements a class with the `pairs()` method that runs through files and returns a KeyValuePair object to be checked with rules. 
 
 ```py
+from whispers.core.utils import KeyValuePair
+
 class PluginName:
-    def pairs(self, file):
-        yield "key", "value"
+  def pairs(self, filepath: Path):
+    yield KeyValuePair(
+      "key",
+      "value",
+      keypath=["node", "path", "to", "key"],
+      file=filepath.as_posix()
+    )
 ```

requirements.txt

@@ -1,17 +1,35 @@
 #
-# This file is autogenerated by pip-compile
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    make freeze
 #
-astroid==2.8.0            # via whispers (setup.py)
-beautifulsoup4==4.10.0     # via whispers (setup.py)
-jproperties==2.1.1        # via whispers (setup.py)
-luhn==0.2.0               # via whispers (setup.py)
-lxml==4.6.3               # via whispers (setup.py)
-python-levenshtein==0.12.2  # via whispers (setup.py)
-pyyaml==5.4.1             # via whispers (setup.py)
-six==1.16.0               # via astroid, jproperties
+astroid==2.8.0
+    # via whispers (setup.py)
+beautifulsoup4==4.10.0
+    # via whispers (setup.py)
+dataclasses==0.6
+    # via whispers (setup.py)
+jproperties==2.1.1
+    # via whispers (setup.py)
+lazy-object-proxy==1.6.0
+    # via astroid
+luhn==0.2.0
+    # via whispers (setup.py)
+lxml==4.6.3
+    # via whispers (setup.py)
+python-levenshtein==0.12.2
+    # via whispers (setup.py)
+pyyaml==5.4.1
+    # via whispers (setup.py)
+six==1.16.0
+    # via jproperties
+soupsieve==2.2.1
+    # via beautifulsoup4
+typing-extensions==3.10.0.2
+    # via astroid
+wrapt==1.12.1
+    # via astroid
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools