Merge pull request #39 from cisagov/lineage/skeleton

⚠️ CONFLICT! Lineage pull request for: skeleton
cisagov · Aug 10, 2021 · 44e0cfb · 44e0cfb
2 parents b668b1c + c72b5a1
commit 44e0cfb
Show file tree

Hide file tree

Showing 16 changed files with 215 additions and 121 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -8,33 +8,102 @@ on:
     types: [apb]
 
 env:
+  CURL_CACHE_DIR: ~/.cache/curl
   IMAGE_NAME: cisagov/gophish-tools
   PIP_CACHE_DIR: ~/.cache/pip
   PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit
+  RUN_TMATE: ${{ secrets.RUN_TMATE }}
 
 jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
+      - uses: cisagov/setup-env-github-action@develop
       - uses: actions/checkout@v2
       - id: setup-python
         uses: actions/setup-python@v2
         with:
           python-version: 3.9
+      # GO_VERSION and GOCACHE are used by the cache task, so the Go
+      # installation must happen before that.
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.16'
+      - name: Store installed Go version
+        run: |
+          echo "GO_VERSION="\
+          "$(go version | sed 's/^go version go\([0-9.]\+\) .*/\1/')" \
+          >> $GITHUB_ENV
+      - name: Lookup Go cache directory
+        id: go-cache
+        run: |
+          echo "::set-output name=dir::$(go env GOCACHE)"
       - uses: actions/cache@v2
         env:
           BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\
-            py${{ steps.setup-python.outputs.python-version }}-"
+            py${{ steps.setup-python.outputs.python-version }}-\
+            go${{ env.GO_VERSION }}-\
+            packer${{ env.PACKER_VERSION }}-\
+            tf${{ env.TERRAFORM_VERSION }}-"
         with:
+          # Note that the .terraform directory IS NOT included in the
+          # cache because if we were caching, then we would need to use
+          # the `-upgrade=true` option. This option blindly pulls down the
+          # latest modules and providers instead of checking to see if an
+          # update is required. That behavior defeats the benefits of caching.
+          # so there is no point in doing it for the .terraform directory.
           path: |
             ${{ env.PIP_CACHE_DIR }}
             ${{ env.PRE_COMMIT_CACHE_DIR }}
+            ${{ env.CURL_CACHE_DIR }}
+            ${{ steps.go-cache.outputs.dir }}
+          # We do not use '**/setup.py' in the cache key so only the 'setup.py'
+          # file in the root of the repository is used. This is in case a Python
+          # package were to have a 'setup.py' as part of its internal codebase.
           key: "${{ env.BASE_CACHE_KEY }}\
             ${{ hashFiles('**/requirements-test.txt') }}-\
             ${{ hashFiles('**/requirements.txt') }}-\
-            ${{ hashFiles('**/.pre-commit-config.yaml') }}"
+            ${{ hashFiles('**/.pre-commit-config.yaml') }}-\
+            ${{ hashFiles('setup.py') }}"
           restore-keys: |
             ${{ env.BASE_CACHE_KEY }}
+      - name: Setup curl cache
+        run: mkdir -p ${{ env.CURL_CACHE_DIR }}
+      - name: Install Packer
+        run: |
+          PACKER_ZIP="packer_${PACKER_VERSION}_linux_amd64.zip"
+          curl --output ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \
+            --time-cond ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \
+            --location \
+            "https://releases.hashicorp.com/packer/${PACKER_VERSION}/${PACKER_ZIP}"
+          sudo unzip -d /opt/packer \
+            ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}"
+          sudo mv /usr/local/bin/packer /usr/local/bin/packer-default
+          sudo ln -s /opt/packer/packer /usr/local/bin/packer
+      - name: Install Terraform
+        run: |
+          TERRAFORM_ZIP="terraform_${TERRAFORM_VERSION}_linux_amd64.zip"
+          curl --output ${{ env.CURL_CACHE_DIR }}/"${TERRAFORM_ZIP}" \
+            --time-cond ${{ env.CURL_CACHE_DIR }}/"${TERRAFORM_ZIP}" \
+            --location \
+            "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/${TERRAFORM_ZIP}"
+          sudo unzip -d /opt/terraform \
+            ${{ env.CURL_CACHE_DIR }}/"${TERRAFORM_ZIP}"
+          sudo mv /usr/local/bin/terraform /usr/local/bin/terraform-default
+          sudo ln -s /opt/terraform/terraform /usr/local/bin/terraform
+      - name: Install shfmt
+        run: go install mvdan.cc/sh/v3/cmd/shfmt@${SHFMT_VERSION}
+      - name: Install Terraform-docs
+        run: |
+          go install \
+            github.com/terraform-docs/terraform-docs@${TERRAFORM_DOCS_VERSION}
+      - name: Find and initialize Terraform directories
+        run: |
+          for path in $(find . -not \( -type d -name ".terraform" -prune \) \
+            -type f -iname "*.tf" -exec dirname "{}" \; | sort -u); do \
+            echo "Initializing '$path'..."; \
+            terraform init -input=false -backend=false "$path"; \
+            done
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -43,6 +112,9 @@ jobs:
         run: pre-commit install-hooks
       - name: Run pre-commit on all files
         run: pre-commit run --all-files
+      - name: Setup tmate debug session
+        uses: mxschmitt/action-tmate@v3
+        if: env.RUN_TMATE
   test:
     runs-on: ubuntu-latest
     strategy:
@@ -60,9 +132,13 @@ jobs:
             py${{ steps.setup-python.outputs.python-version }}-"
         with:
           path: ${{ env.PIP_CACHE_DIR }}
+          # We do not use '**/setup.py' in the cache key so only the 'setup.py'
+          # file in the root of the repository is used. This is in case a Python
+          # package were to have a 'setup.py' as part of its internal codebase.
           key: "${{ env.BASE_CACHE_KEY }}\
             ${{ hashFiles('**/requirements-test.txt') }}-\
-            ${{ hashFiles('**/requirements.txt') }}"
+            ${{ hashFiles('**/requirements.txt') }}-\
+            ${{ hashFiles('setup.py') }}"
           restore-keys: |
             ${{ env.BASE_CACHE_KEY }}
       - name: Install dependencies
@@ -81,6 +157,9 @@ jobs:
           COVERALLS_SERVICE_NAME: github
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         if: success()
+      - name: Setup tmate debug session
+        uses: mxschmitt/action-tmate@v3
+        if: env.RUN_TMATE
   coveralls-finish:
     runs-on: ubuntu-latest
     needs: test
@@ -96,9 +175,13 @@ jobs:
             py${{ steps.setup-python.outputs.python-version }}-"
         with:
           path: ${{ env.PIP_CACHE_DIR }}
+          # We do not use '**/setup.py' in the cache key so only the 'setup.py'
+          # file in the root of the repository is used. This is in case a Python
+          # package were to have a 'setup.py' as part of its internal codebase.
           key: "${{ env.BASE_CACHE_KEY }}\
             ${{ hashFiles('**/requirements-test.txt') }}-\
-            ${{ hashFiles('**/requirements.txt') }}"
+            ${{ hashFiles('**/requirements.txt') }}-\
+            ${{ hashFiles('setup.py') }}"
           restore-keys: |
             ${{ env.BASE_CACHE_KEY }}
       - name: Install dependencies
@@ -109,6 +192,9 @@ jobs:
         run: coveralls --finish
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Setup tmate debug session
+        uses: mxschmitt/action-tmate@v3
+        if: env.RUN_TMATE
   build:
     runs-on: ubuntu-latest
     needs: [lint, test]
@@ -127,8 +213,12 @@ jobs:
             py${{ steps.setup-python.outputs.python-version }}-"
         with:
           path: ${{ env.PIP_CACHE_DIR }}
+          # We do not use '**/setup.py' in the cache key so only the 'setup.py'
+          # file in the root of the repository is used. This is in case a Python
+          # package were to have a 'setup.py' as part of its internal codebase.
           key: "${{ env.BASE_CACHE_KEY }}\
-            ${{ hashFiles('**/requirements.txt') }}"
+            ${{ hashFiles('**/requirements.txt') }}-\
+            ${{ hashFiles('setup.py') }}"
           restore-keys: |
             ${{ env.BASE_CACHE_KEY }}
       - name: Install dependencies
@@ -159,3 +249,6 @@ jobs:
         with:
           name: dist-${{ matrix.python-version }}
           path: dist
+      - name: Setup tmate debug session
+        uses: mxschmitt/action-tmate@v3
+        if: env.RUN_TMATE
diff --git a/.mdl_config.json b/.mdl_config.json
diff --git a/.mdl_config.yaml b/.mdl_config.yaml
@@ -0,0 +1,40 @@
+---
+
+# Default state for all rules
+default: true
+
+# MD003/heading-style/header-style - Heading style
+MD003:
+  # Enforce the ATX-closed style of header
+  style: "atx_closed"
+
+# MD004/ul-style - Unordered list style
+MD004:
+  # Enforce dashes for unordered lists
+  style: "dash"
+
+# MD013/line-length - Line length
+MD013:
+  # Do not enforce for code blocks
+  code_blocks: false
+  # Do not enforce for tables
+  tables: false
+
+# MD024/no-duplicate-heading/no-duplicate-header - Multiple headings with the
+# same content
+MD024:
+  # Allow headers with the same content as long as they are not in the same
+  # parent heading
+  allow_different_nesting: true
+
+# MD029/ol-prefix - Ordered list item prefix
+MD029:
+  # Enforce the `1.` style for ordered lists
+  style: "one"
+
+# MD033/no-inline-html - Inline HTML
+MD033:
+  # The h1 and img elements are allowed to permit header images
+  allowed_elements:
+    - h1
+    - img
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -35,9 +35,9 @@ repos:
     hooks:
       - id: markdownlint
         args:
-          - --config=.mdl_config.json
+          - --config=.mdl_config.yaml
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v2.3.0
+    rev: v2.3.2
     hooks:
       - id: prettier
   - repo: https://github.com/adrienverge/yamllint
@@ -47,14 +47,27 @@ repos:
         args:
           - --strict
 
+  # pre-commit hooks
+  - repo: https://github.com/pre-commit/pre-commit
+    rev: v2.13.0
+    hooks:
+      - id: validate_manifest
+
   # Shell script hooks
-  - repo: https://github.com/lovesegfault/beautysh
-    rev: v6.1.0
+  - repo: https://github.com/cisagov/pre-commit-shfmt
+    rev: v0.0.2
     hooks:
-      - id: beautysh
+      - id: shfmt
         args:
-          - --indent-size
+          # Indent by two spaces
+          - -i
           - '2'
+          # Binary operators may start a line
+          - -bn
+          # Switch cases are indented
+          - -ci
+          # Redirect operators are followed by a space
+          - -sr
   - repo: https://github.com/detailyang/pre-commit-shell
     rev: 1.0.5
     hooks:
@@ -78,7 +91,7 @@ repos:
         name: bandit (everything else)
         exclude: tests
   - repo: https://github.com/psf/black
-    rev: 21.5b2
+    rev: 21.7b0
     hooks:
       - id: black
   - repo: https://gitlab.com/pycqa/flake8
@@ -88,15 +101,19 @@ repos:
         additional_dependencies:
           - flake8-docstrings
   - repo: https://github.com/PyCQA/isort
-    rev: 5.8.0
+    rev: 5.9.2
     hooks:
       - id: isort
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.812
+    rev: v0.910
     hooks:
       - id: mypy
+        additional_dependencies:
+          - types-mock
+          - types-pytz
+          - types-requests
   - repo: https://github.com/asottile/pyupgrade
-    rev: v2.19.1
+    rev: v2.21.2
     hooks:
       - id: pyupgrade
 
@@ -133,7 +150,7 @@ repos:
 
   # Docker hooks
   - repo: https://github.com/IamTheFij/docker-pre-commit
-    rev: v2.0.0
+    rev: v2.0.1
     hooks:
       - id: docker-compose-check
 

diff --git a/README.md b/README.md
@@ -16,18 +16,18 @@ campaign assessors to simplify the process of managing GoPhish campaigns.
 
 ## Scripts ##
 
-* `gophish-cleaner` - Removes an assessment or elements of an assessment
+- `gophish-cleaner` - Removes an assessment or elements of an assessment
   in GoPhish.
-* `gophish-complete` - Completes a campaign in GoPhish and/or outputs a
+- `gophish-complete` - Completes a campaign in GoPhish and/or outputs a
   GoPhish campaign summary.
-* `gophish-export` - Exports all the data from an assessment within GoPhish
+- `gophish-export` - Exports all the data from an assessment within GoPhish
   into a single JSON file. In addition, user report JSONs for each campaign in
   an assessment will also be generated.
-* `gophish-import` - Imports an assessment JSON file into GoPhish.
-* `gophish-test` - Sends a duplicate assessment from GoPhish to custom
+- `gophish-import` - Imports an assessment JSON file into GoPhish.
+- `gophish-test` - Sends a duplicate assessment from GoPhish to custom
   targets as a test.
-* `pca-wizard` - Creates an assessment JSON file via an interactive "wizard".
-* `pca-wizard-templates` - Generates templates for files needed when creating
+- `pca-wizard` - Creates an assessment JSON file via an interactive "wizard".
+- `pca-wizard-templates` - Generates templates for files needed when creating
   an assessment JSON with `pca-wizard`.
 
 ## Usage ##

diff --git a/bump_version.sh b/bump_version.sh
@@ -8,16 +8,15 @@ set -o pipefail
 
 HELP_INFORMATION="bump_version.sh (assessment|project|templates|tools) (show|major|minor|patch|prerelease|build|finalize)"
 
-if [ $# -ne 2 ]
-then
+if [ $# -ne 2 ]; then
   echo "$HELP_INFORMATION"
 else
   case $1 in
     project)
       VERSION_FILE=src/_version.py
       old_version=$(sed -n "s/^__version__ = \"\(.*\)\"$/\1/p" "$VERSION_FILE")
       ;;
-    assessment|templates|tools)
+    assessment | templates | tools)
       VERSION_FILE=src/$1/_version.py
       old_version=$(sed -n "s/^__version__ = \"\(.*\)\"$/\1/p" "$VERSION_FILE")
       ;;
@@ -27,7 +26,7 @@ else
       ;;
   esac
   case $2 in
-    major|minor|patch|prerelease|build)
+    major | minor | patch | prerelease | build)
       new_version=$(python -c "import semver; print(semver.bump_$2('$old_version'))")
       echo Changing "$1" version from "$old_version" to "$new_version"
       # A temp file is used to provide compatability with macOS development