Update to skeleton-aws-lambda, Pare CODEOWNERS, and Add Lineage Configuration

.bandit.yml

@@ -11,4 +11,4 @@ tests:
 # - B102
 
 skips:
-  - B101 # skip "assert used" check since assertions are required in pytests
+  - B101  # skip "assert used" check since assertions are required in pytests

.github/CODEOWNERS

@@ -4,4 +4,4 @@
 # the repo. Unless a later match takes precedence,
 # these owners will be requested for review when someone
 # opens a pull request.
-*       @dav3r @felddy @jsf9k @mcdonnnj @cisagov/team-ois @mzack5020
+*       @mcdonnnj @mzack5020

.github/lineage.yml

@@ -0,0 +1,6 @@
+---
+version: "1"
+
+lineage:
+  skeleton:
+    remote-url: https://github.com/cisagov/skeleton-aws-lambda.git

.github/workflows/build.yml

@@ -1,7 +1,11 @@
 ---
 name: build
 
-on: [push]
+on:
+  push:
+  pull_request:
+  repository_dispatch:
+    types: [apb]
 
 env:
   PIP_CACHE_DIR: ~/.cache/pip
@@ -11,51 +15,63 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
       - uses: actions/setup-python@v1
         with:
           python-version: 3.8
+      - name: Store installed Python version
+        run: |
+          echo "::set-env name=PY_VERSION::"\
+          "$(python -c "import platform;print(platform.python_version())")"
       - name: Cache pip test requirements
         uses: actions/cache@v1
         with:
           path: ${{ env.PIP_CACHE_DIR }}
-          key: "${{ runner.os }}-pip-test-\
+          key: "${{ runner.os }}-pip-test-py${{ env.PY_VERSION }}-\
             ${{ hashFiles('**/requirements-test.txt') }}"
           restore-keys: |
+            ${{ runner.os }}-pip-test-py${{ env.PY_VERSION }}-
             ${{ runner.os }}-pip-test-
             ${{ runner.os }}-pip-
       - name: Cache pre-commit hooks
         uses: actions/cache@v1
         with:
           path: ${{ env.PRE_COMMIT_CACHE_DIR }}
-          key: "${{ runner.os }}-pre-commit-\
+          key: "${{ runner.os }}-pre-commit-py${{ env.PY_VERSION }}-\
             ${{ hashFiles('**/.pre-commit-config.yaml') }}"
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install --upgrade -r requirements-test.txt
-      - name: Run linters on all files
+          pip install --upgrade --requirement requirements-test.txt
+      - name: Set up pre-commit hook environments
+        run: pre-commit install-hooks
+      - name: Run pre-commit on all files
         run: pre-commit run --all-files
   test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
       - uses: actions/setup-python@v1
         with:
           python-version: 3.8
+      - name: Store installed Python version
+        run: |
+          echo "::set-env name=PY_VERSION::"\
+          "$(python -c "import platform;print(platform.python_version())")"
       - name: Cache pip test requirements
         uses: actions/cache@v1
         with:
           path: ${{ env.PIP_CACHE_DIR }}
-          key: "${{ runner.os }}-pip-test-\
+          key: "${{ runner.os }}-pip-test-py${{ env.PY_VERSION }}-\
             ${{ hashFiles('**/requirements-test.txt') }}"
           restore-keys: |
+            ${{ runner.os }}-pip-test-py${{ env.PY_VERSION }}-
             ${{ runner.os }}-pip-test-
             ${{ runner.os }}-pip-
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install --upgrade -r requirements-test.txt
+          pip install --upgrade --requirement requirements-test.txt
       - name: Run tests
         env:
           RELEASE_TAG: ${{ github.event.release.tag_name }}
@@ -69,23 +85,28 @@ jobs:
     runs-on: ubuntu-latest
     needs: [lint, test]
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
       - uses: actions/setup-python@v1
         with:
           python-version: 3.8
+      - name: Store installed Python version
+        run: |
+          echo "::set-env name=PY_VERSION::"\
+          "$(python -c "import platform;print(platform.python_version())")"
       - name: Cache pip build requirements
         uses: actions/cache@v1
         with:
           path: ${{ env.PIP_CACHE_DIR }}
-          key: "${{ runner.os }}-pip-build-\
+          key: "${{ runner.os }}-pip-build-py${{ env.PY_VERSION }}-\
             ${{ hashFiles('**/requirements.txt') }}"
           restore-keys: |
+            ${{ runner.os }}-pip-build-py${{ env.PY_VERSION }}-
             ${{ runner.os }}-pip-build-
             ${{ runner.os }}-pip-
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip wheel
-          pip install --upgrade -r requirements.txt
+          pip install --upgrade --requirement requirements.txt
       - name: Build environment
         run: docker-compose build
       - name: Generate lambda zip

.gitignore

@@ -1,8 +1,9 @@
-.python-version
-.DS_Store
 *.egg-info
-__pycache__
-findings-data-import.zip
-.vscode
 .coverage
+.DS_Store
+.mypy_cache
 .pytest_cache
+.python-version
+.vscode
+__pycache__
+findings-data-import.zip

.mdl_config.json

@@ -3,5 +3,8 @@
     "code_blocks": false,
     "tables": false
   },
+  "MD024": {
+    "allow_different_nesting": true
+  },
   "default": true
 }

.pre-commit-config.yaml

@@ -5,7 +5,7 @@ default_language_version:
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.4.0
+    rev: v2.5.0
     hooks:
       - id: check-executables-have-shebangs
       - id: check-json
@@ -27,27 +27,27 @@ repos:
       - id: requirements-txt-fixer
       - id: trailing-whitespace
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.19.0
+    rev: v0.23.0
     hooks:
       - id: markdownlint
         args:
           - --config=.mdl_config.json
   - repo: https://github.com/adrienverge/yamllint
-    rev: v1.18.0
+    rev: v1.23.0
     hooks:
       - id: yamllint
   - repo: https://github.com/detailyang/pre-commit-shell
     rev: 1.0.5
     hooks:
       - id: shell-lint
   - repo: https://gitlab.com/pycqa/flake8
-    rev: 3.7.9
+    rev: 3.8.1
     hooks:
       - id: flake8
         additional_dependencies:
           - flake8-docstrings
   - repo: https://github.com/asottile/pyupgrade
-    rev: v1.25.1
+    rev: v2.4.1
     hooks:
       - id: pyupgrade
   # Run bandit on "tests" tree with a configuration
@@ -71,31 +71,47 @@ repos:
     hooks:
       - id: black
   - repo: https://github.com/asottile/seed-isort-config
-    rev: v1.9.3
+    rev: v2.1.1
     hooks:
       - id: seed-isort-config
-  - repo: https://github.com/pre-commit/mirrors-isort
-    # pick the isort version you'd like to use from
-    # https://github.com/pre-commit/mirrors-isort/releases
-    rev: v4.3.21
+  - repo: https://github.com/timothycrosley/isort
+    rev: 4.3.21
     hooks:
       - id: isort
   - repo: https://github.com/ansible/ansible-lint.git
-    rev: v4.1.1a5
+    rev: v4.3.0a0
     hooks:
       - id: ansible-lint
-        # files: molecule/default/playbook.yml
+      # files: molecule/default/playbook.yml
   - repo: https://github.com/antonbabenko/pre-commit-terraform.git
-    rev: v1.12.0
+    rev: v1.30.0
     hooks:
       - id: terraform_fmt
-      - id: terraform_validate_no_variables
-      - id: terraform_docs
+      # There are ongoing issues with how this command works. This issue
+      # documents the core issue:
+      # https://github.com/hashicorp/terraform/issues/21408
+      # We have seen issues primarily with proxy providers and Terraform code
+      # that uses remote state. The PR
+      # https://github.com/hashicorp/terraform/pull/24887
+      # has been approved and is part of the 0.13 release to resolve the issue
+      # with remote states.
+      # The PR
+      # https://github.com/hashicorp/terraform/pull/24896
+      # is a proprosed fix to deal with `terraform validate` with proxy
+      # providers (among other configurations).
+      # We have decided to disable the terraform_validate hook until the issues
+      # above have been resolved, which we hope will be with the release of
+      # Terraform 0.13.
+      # - id: terraform_validate
   - repo: https://github.com/IamTheFij/docker-pre-commit
-    rev: v1.0.0
+    rev: v1.0.1
     hooks:
       - id: docker-compose-check
   - repo: https://github.com/prettier/prettier
-    rev: 1.19.1
+    rev: 2.0.5
     hooks:
       - id: prettier
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.770
+    hooks:
+      - id: mypy

CONTRIBUTING.md

@@ -56,10 +56,31 @@ eval "$(pyenv init -)"
 eval "$(pyenv virtualenv-init -)"
 ```
 
-For Linux (or on the Mac, if you don't want to use `brew`) you can use
+For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you
+don't want to use `brew`) you can use
 [pyenv/pyenv-installer](https://github.com/pyenv/pyenv-installer) to
-install the necessary tools.  When you are finished you will need to
-add the same two lines above to your profile.
+install the necessary tools. Before running this ensure that you have
+installed the prerequisites for your platform according to the
+[`pyenv` wiki
+page](https://github.com/pyenv/pyenv/wiki/common-build-problems).
+
+On WSL you should treat your platform as whatever Linux distribution
+you've chosen to install.
+
+Once you have installed `pyenv` you will need to add the following
+lines to your `.bashrc`:
+
+```bash
+export PATH="$PATH:$HOME/.pyenv/bin"
+eval "$(pyenv init -)"
+eval "$(pyenv virtualenv-init -)"
+```
+
+If you are using a shell other than `bash` you should follow the
+instructions that the `pyenv-installer` script outputs.
+
+You will need to reload your shell for these changes to take effect so
+you can begin to use `pyenv`.
 
 For a list of Python versions that are already installed and ready to
 use with `pyenv`, use the command `pyenv versions`.  To see a list of
@@ -81,7 +102,7 @@ commands:
 cd findings-data-import-lambda
 pyenv virtualenv <python_version_to_use> findings-data-import-lambda
 pyenv local findings-data-import-lambda
-pip install -r requirements-dev.txt
+pip install --requirement requirements-dev.txt
 ```
 
 #### Installing the pre-commit hook ####

Dockerfile

@@ -1,9 +1,10 @@
 FROM lambci/lambda:build-python3.8
-MAINTAINER Matthew Zackschewski <matthew.zackschewski@trio.dhs.gov>
+LABEL maintainer="Matthew Zackschewski <matthew.zackschewski@trio.dhs.gov>"
+LABEL vendor="Cyber and Infrastructure Security Agency"
 
 COPY build.sh .
 
-# Files needed to install local fdi module
+# Files needed to install local fdi module.
 COPY setup.py .
 COPY requirements.txt .
 COPY README.md .

README.md

@@ -56,6 +56,11 @@ image is very fast (due to Docker's caching) if the script has not changed, it
 is a very good idea to _always_ run the `docker-compose build` step when
 using this tool.
 
+## Contributing ##
+
+We welcome contributions!  Please see [here](CONTRIBUTING.md) for
+details.
+
 ## License ##
 
 This project is in the worldwide [public domain](LICENSE).

build.sh

@@ -26,17 +26,17 @@ python -m venv --system-site-packages $VENV_DIR
 source $VENV_DIR/bin/activate
 
 ###
-# Update pip and setuptools
+# Upgrade pip.
 ###
 pip install --upgrade pip
 
 ###
-# Install local findings data import (fdi) module and requirements
+# Install local findings data import (fdi) module and requirements.
 ###
-pip install -r requirements.txt
+pip install --requirement requirements.txt
 
 ###
-# Leave the Python virtual environment
+# Leave the Python virtual environment.
 #
 # Note that we have to turn off nounset before running deactivate,
 # since otherwise we get an error that states "/venv/bin/activate:
@@ -47,21 +47,21 @@ deactivate
 set -o nounset
 
 ###
-# Set up the build directory
+# Set up the build directory.
 ###
 BUILD_DIR=/build
 
 ###
-# Copy all packages, including any hidden dotfiles.  Also copy the
-# local fdi package and the Lambda handler.
+# Copy all packages, including any hidden dotfiles. Also copy the
+# local fdi package and the lambda handler.
 ###
 cp -rT $VENV_DIR/lib/python3.8/site-packages/ $BUILD_DIR
 cp -rT $VENV_DIR/lib64/python3.8/site-packages/ $BUILD_DIR
 cp -r fdi $BUILD_DIR
 cp lambda_handler.py $BUILD_DIR
 
 ###
-# Zip it all up
+# Zip it all up.
 ###
 OUTPUT_DIR=/output
 if [ ! -d $OUTPUT_DIR ]
@@ -73,5 +73,6 @@ if [ -e $OUTPUT_DIR/$ZIP_FILE ]
 then
     rm $OUTPUT_DIR/$ZIP_FILE
 fi
+
 cd $BUILD_DIR
 zip -rq9 $OUTPUT_DIR/$ZIP_FILE .

docker-compose.yml

@@ -3,6 +3,6 @@ version: '3.2'
 services:
   build_findings_data_import_lambda:
     build: .
-    image: 'dhsncats/build_data_ingestion_lambda'
+    image: 'cisagov/build_findings_data_import_lambda'
     volumes:
       - .:/output