Merge pull request #8 from PaperMtn/release/2.0.0

Release/2.0.0

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,42 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. iOS]
+ - Browser [e.g. chrome, safari]
+ - Version [e.g. 22]
+
+**Smartphone (please complete the following information):**
+ - Device: [e.g. iPhone6]
+ - OS: [e.g. iOS8.1]
+ - Browser [e.g. stock browser, safari]
+ - Version [e.g. 22]
+
+**Additional context**
+<<<<<<< HEAD
+Add any other context about the problem here.
+=======
+Add any other context about the problem here.
+>>>>>>> 500caddadff50ff27d7100a970f2971515df6f4b

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,24 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+<<<<<<< HEAD
+Add any other context or screenshots about the feature request here.
+=======
+Add any other context or screenshots about the feature request here.
+>>>>>>> 500caddadff50ff27d7100a970f2971515df6f4b

.github/workflows/dockerpublish.yml

@@ -0,0 +1,37 @@
+name: Docker Image Publish
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  github-cache:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfile
+          push: true
+          tags: papermountain/gitlab-watchman:latest
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache

.github/workflows/pythonpackage.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8]
+        python-version: ['3.7', '3.8', '3.9', '3.10']
 
     steps:
     - uses: actions/checkout@v2
@@ -36,12 +36,12 @@ jobs:
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
     - name: Test setup & install
       run: |
-        pip install wheel setuptools requests colorama termcolor PyYAML
-        python setup.py sdist bdist_wheel
-        pip install dist/*.whl
+        pip install build
+        python3 -m build
+        python3 -m pip install dist/*.whl
     - name: Test rules
       run: |
-        python3 -m unittest tests/test_rules.py
+        python3 -m unittest tests/test_signatures.py
     - name: Test run
       run: |
         gitlab-watchman --version
@@ -52,7 +52,7 @@ jobs:
     runs-on: windows-latest
     strategy:
       matrix:
-        python-version: [3.5, 3.6, 3.7, 3.8]
+        python-version: ['3.7', '3.8', '3.9', '3.10']
 
     steps:
     - uses: actions/checkout@v2
@@ -63,10 +63,10 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install wheel setuptools requests colorama termcolor
+        pip install build
     - name: Test setup & install
       run: |
-        python setup.py sdist bdist_wheel
+        python -m build
         pip install --find-links=dist\ gitlab-watchman
     - name: Test run
       run: |

.github/workflows/pythonpublish.yml

@@ -20,12 +20,11 @@ jobs:
         python-version: '3.x'
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
-        pip install setuptools wheel twine requests colorama termcolor PyYAML
+        python3 -m pip install --upgrade pip twine build
     - name: Build and publish
       env:
         TWINE_USERNAME: ${{ '__token__' }}
         TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
       run: |
-        python setup.py sdist bdist_wheel
+        python3 -m build
         twine upload dist/*

.gitignore

@@ -123,8 +123,3 @@ venv.bak/
 .mypy_cache/
 .dmypy.json
 dmypy.json
-
-# package related
-config/
-*.txt
-*.csv

CHANGELOG.md

@@ -1,3 +1,24 @@
+## 2.0.0 - 2022-04-01
+### Added:
+- New scopes for finding exposed data in:
+  - notes
+  - snippets
+- Docker image now available from the Docker hub, or by building from source. (Credit [@adioss](https://github.com/adioss) for the inspiration)
+- Complete rewrite of the codebase to make searching faster and more efficient.
+  - More modern packaging and distribution.
+- Logs now include more data
+- Additional signatures added to find more leaked data
+- Updated logo to play nicely with dark mode displays
+
+### Removed:
+
+- Logging to file and TCP stream - logs to stdout like a true 12 factor app. Reroute stdout as you see fit. --output 
+- .conf file for configuration options. Pass the environment variables `GITLAB_WATCHMAN_TOKEN` and `GITLAB_WATCHMAN_URL`
+
+**Breaking changes:**
+- The --output flag is no longer required, and therefore not supported
+
+
 ## 1.4.0 - 2020-12-24
 ### Added:
 - Refactor of rules into directories for easier management
@@ -18,7 +39,7 @@
 ## 1.3.0 - 2020-12-12
 ### Added:
 - Add more information about the namespaces a project is in to logs
-- Added details owner of that namespace, for groups and users
+- Added owner details of that namespace, for groups and users
 - Time based searching now looks at the time a file was committed, not when a project was active, which greatly reduces multiples of the same detection because a project is active but a file has not been modified.
 - Rules added:
     - SSH private keys

CODE_OF_CONDUCT.md

@@ -0,0 +1,128 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+papermtn@protonmail.com.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at
+https://www.contributor-covenant.org/translations.

CONTRIBUTING.md

@@ -0,0 +1,26 @@
+# Contributing
+
+Thanks for being interested in contributing to GitLab Watchman. I'm always looking for more contributions towards the project.
+
+The two main areas where you can contribute are:
+- Signature files
+- Additional functionality
+
+
+## Adding new signatures
+GitLab Watchman runs using YAML signature files that are stored in the `signature` directory. They define what to search Slack for, and are the heart of the application.
+
+Instructions on how to create your own signature files can be found in `docs\signatures.md`
+
+If you do write your own signatures, please contribute them to the project by creating a pull request.
+
+
+## Additional functionality
+You can make recommendations for new functionality via raising issues using the feature request template. Even better, you could contribute the additional functionality yourself and create a pull request for the changes to be added to a future release.
+
+## Style
+
+GitLab Watchman follows Google style guidelines mostly, but the main thing is to stay consistent to the style already in use.
+
+## Documentation
+Currently, documentation and additional resources are kept on [my blog](https://papermtn.co.uk/category/tools/gitlab-watchman/) 

Dockerfile

@@ -0,0 +1,20 @@
+FROM alpine/git AS initlayer
+WORKDIR /workdir
+RUN git clone https://github.com/PaperMtn/gitlab-watchman.git
+
+FROM python:buster
+RUN addgroup --gid 1000 gitlab-watchman
+RUN useradd -u 1000 -g 1000 gitlab-watchman
+RUN mkdir /home/gitlab-watchman
+COPY --from=initlayer /workdir/gitlab-watchman /home/gitlab-watchman
+RUN chown -R gitlab-watchman: /home/gitlab-watchman
+WORKDIR /home/gitlab-watchman
+
+RUN python3 -m pip install --upgrade pip
+RUN python3 -m pip install requests build PyYAML
+RUN python3 -m build
+RUN python3 -m pip install dist/*.whl
+
+USER gitlab-watchman
+
+ENTRYPOINT ["/usr/local/bin/gitlab-watchman"]

MANIFEST.in

@@ -1,3 +1,3 @@
 include *.txt
-recursive-include gitlab_watchman *.yaml *.yml
-recursive-exclude gitlab_watchman test*.yaml test*.yml
+recursive-include src/signatures *.yaml *.yml
+recursive-exclude src/signatures test*.yaml test*.yml