diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 695e8cc..fb65a3a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,32 +8,97 @@ on: types: [apb] env: + CURL_CACHE_DIR: ~/.cache/curl PIP_CACHE_DIR: ~/.cache/pip PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit + RUN_TMATE: ${{ secrets.RUN_TMATE }} jobs: lint: runs-on: ubuntu-latest steps: + - uses: cisagov/setup-env-github-action@develop - uses: actions/checkout@v2 - id: setup-python uses: actions/setup-python@v2 with: python-version: 3.9 + # GO_VERSION and GOCACHE are used by the cache task, so the Go + # installation must happen before that. + - uses: actions/setup-go@v2 + with: + go-version: '1.16' + - name: Store installed Go version + run: | + echo "GO_VERSION="\ + "$(go version | sed 's/^go version go\([0-9.]\+\) .*/\1/')" \ + >> $GITHUB_ENV + - name: Lookup Go cache directory + id: go-cache + run: | + echo "::set-output name=dir::$(go env GOCACHE)" - uses: actions/cache@v2 env: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ - py${{ steps.setup-python.outputs.python-version }}-" + py${{ steps.setup-python.outputs.python-version }}-\ + go${{ env.GO_VERSION }}-\ + packer${{ env.PACKER_VERSION }}-\ + tf${{ env.TERRAFORM_VERSION }}-" with: + # Note that the .terraform directory IS NOT included in the + # cache because if we were caching, then we would need to use + # the `-upgrade=true` option. This option blindly pulls down the + # latest modules and providers instead of checking to see if an + # update is required. That behavior defeats the benefits of caching. + # so there is no point in doing it for the .terraform directory. path: | ${{ env.PIP_CACHE_DIR }} ${{ env.PRE_COMMIT_CACHE_DIR }} + ${{ env.CURL_CACHE_DIR }} + ${{ steps.go-cache.outputs.dir }} key: "${{ env.BASE_CACHE_KEY }}\ ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/requirements.txt') }}-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" restore-keys: | ${{ env.BASE_CACHE_KEY }} + - name: Setup curl cache + run: mkdir -p ${{ env.CURL_CACHE_DIR }} + - name: Install Packer + run: | + PACKER_ZIP="packer_${PACKER_VERSION}_linux_amd64.zip" + curl --output ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ + --time-cond ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ + --location \ + "https://releases.hashicorp.com/packer/${PACKER_VERSION}/${PACKER_ZIP}" + sudo unzip -d /opt/packer \ + ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" + sudo mv /usr/local/bin/packer /usr/local/bin/packer-default + sudo ln -s /opt/packer/packer /usr/local/bin/packer + - name: Install Terraform + run: | + TERRAFORM_ZIP="terraform_${TERRAFORM_VERSION}_linux_amd64.zip" + curl --output ${{ env.CURL_CACHE_DIR }}/"${TERRAFORM_ZIP}" \ + --time-cond ${{ env.CURL_CACHE_DIR }}/"${TERRAFORM_ZIP}" \ + --location \ + "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/${TERRAFORM_ZIP}" + sudo unzip -d /opt/terraform \ + ${{ env.CURL_CACHE_DIR }}/"${TERRAFORM_ZIP}" + sudo mv /usr/local/bin/terraform /usr/local/bin/terraform-default + sudo ln -s /opt/terraform/terraform /usr/local/bin/terraform + - name: Install shfmt + run: go install mvdan.cc/sh/v3/cmd/shfmt@${SHFMT_VERSION} + - name: Install Terraform-docs + run: | + go install \ + github.com/terraform-docs/terraform-docs@${TERRAFORM_DOCS_VERSION} + - name: Find and initialize Terraform directories + run: | + for path in $(find . -not \( -type d -name ".terraform" -prune \) \ + -type f -iname "*.tf" -exec dirname "{}" \; | sort -u); do \ + echo "Initializing '$path'..."; \ + terraform init -input=false -backend=false "$path"; \ + done - name: Install dependencies run: | python -m pip install --upgrade pip @@ -42,6 +107,9 @@ jobs: run: pre-commit install-hooks - name: Run pre-commit on all files run: pre-commit run --all-files + - name: Setup tmate debug session + uses: mxschmitt/action-tmate@v3 + if: env.RUN_TMATE test: runs-on: ubuntu-latest strategy: @@ -80,6 +148,9 @@ jobs: COVERALLS_SERVICE_NAME: github GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} if: success() + - name: Setup tmate debug session + uses: mxschmitt/action-tmate@v3 + if: env.RUN_TMATE coveralls-finish: runs-on: ubuntu-latest needs: test @@ -108,6 +179,9 @@ jobs: run: coveralls --finish env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Setup tmate debug session + uses: mxschmitt/action-tmate@v3 + if: env.RUN_TMATE build: runs-on: ubuntu-latest needs: [lint, test] @@ -141,3 +215,6 @@ jobs: with: name: dist-${{ matrix.python-version }} path: dist + - name: Setup tmate debug session + uses: mxschmitt/action-tmate@v3 + if: env.RUN_TMATE diff --git a/.mdl_config.json b/.mdl_config.json deleted file mode 100644 index 7a6f3f8..0000000 --- a/.mdl_config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "MD013": { - "code_blocks": false, - "tables": false - }, - "MD024": { - "allow_different_nesting": true - }, - "default": true -} diff --git a/.mdl_config.yaml b/.mdl_config.yaml new file mode 100644 index 0000000..a2f08f3 --- /dev/null +++ b/.mdl_config.yaml @@ -0,0 +1,40 @@ +--- + +# Default state for all rules +default: true + +# MD003/heading-style/header-style - Heading style +MD003: + # Enforce the ATX-closed style of header + style: "atx_closed" + +# MD004/ul-style - Unordered list style +MD004: + # Enforce dashes for unordered lists + style: "dash" + +# MD013/line-length - Line length +MD013: + # Do not enforce for code blocks + code_blocks: false + # Do not enforce for tables + tables: false + +# MD024/no-duplicate-heading/no-duplicate-header - Multiple headings with the +# same content +MD024: + # Allow headers with the same content as long as they are not in the same + # parent heading + allow_different_nesting: true + +# MD029/ol-prefix - Ordered list item prefix +MD029: + # Enforce the `1.` style for ordered lists + style: "one" + +# MD033/no-inline-html - Inline HTML +MD033: + # The h1 and img elements are allowed to permit header images + allowed_elements: + - h1 + - img diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4323029..db9f507 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,9 +35,9 @@ repos: hooks: - id: markdownlint args: - - --config=.mdl_config.json + - --config=.mdl_config.yaml - repo: https://github.com/pre-commit/mirrors-prettier - rev: v2.3.0 + rev: v2.3.2 hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint @@ -47,14 +47,27 @@ repos: args: - --strict + # pre-commit hooks + - repo: https://github.com/pre-commit/pre-commit + rev: v2.13.0 + hooks: + - id: validate_manifest + # Shell script hooks - - repo: https://github.com/lovesegfault/beautysh - rev: v6.1.0 + - repo: https://github.com/cisagov/pre-commit-shfmt + rev: v0.0.2 hooks: - - id: beautysh + - id: shfmt args: - - --indent-size + # Indent by two spaces + - -i - '2' + # Binary operators may start a line + - -bn + # Switch cases are indented + - -ci + # Redirect operators are followed by a space + - -sr - repo: https://github.com/detailyang/pre-commit-shell rev: 1.0.5 hooks: @@ -78,7 +91,7 @@ repos: name: bandit (everything else) exclude: tests - repo: https://github.com/psf/black - rev: 21.5b2 + rev: 21.7b0 hooks: - id: black - repo: https://gitlab.com/pycqa/flake8 @@ -88,15 +101,17 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/PyCQA/isort - rev: 5.8.0 + rev: 5.9.2 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.812 + rev: v0.910 hooks: - id: mypy + additional_dependencies: + - types-setuptools - repo: https://github.com/asottile/pyupgrade - rev: v2.19.1 + rev: v2.21.2 hooks: - id: pyupgrade @@ -133,7 +148,7 @@ repos: # Docker hooks - repo: https://github.com/IamTheFij/docker-pre-commit - rev: v2.0.0 + rev: v2.0.1 hooks: - id: docker-compose-check diff --git a/bump_version.sh b/bump_version.sh index c0bf25c..e1324b8 100755 --- a/bump_version.sh +++ b/bump_version.sh @@ -12,12 +12,11 @@ HELP_INFORMATION="bump_version.sh (show|major|minor|patch|prerelease|build|final old_version=$(sed -n "s/^__version__ = \"\(.*\)\"$/\1/p" $VERSION_FILE) -if [ $# -ne 1 ] -then +if [ $# -ne 1 ]; then echo "$HELP_INFORMATION" else case $1 in - major|minor|patch|prerelease|build) + major | minor | patch | prerelease | build) new_version=$(python -c "import semver; print(semver.bump_$1('$old_version'))") echo Changing version from "$old_version" to "$new_version" # A temp file is used to provide compatability with macOS development diff --git a/setup-env b/setup-env index 1579e04..6dd2079 100755 --- a/setup-env +++ b/setup-env @@ -4,14 +4,15 @@ set -o nounset set -o errexit set -o pipefail -USAGE=$(cat << 'END_OF_LINE' +USAGE=$( + cat << 'END_OF_LINE' Configure a developement environment for this repository. It does the following: - Verifies pyenv and pyenv-virtualenv are installed. - Creates a Python virtual environment. - Configures the activation of the virtual enviroment for the repo directory. - - Installs the requirements needed for development. + - Installs the requirements needed for development (including mypy type stubs). - Installs git pre-commit hooks. - Configures git upstream remote "lineage" repositories. @@ -35,17 +36,17 @@ FORCE=0 PARAMS="" # Parse command line arguments -while (( "$#" )); do +while (("$#")); do case "$1" in - -f|--force) + -f | --force) FORCE=1 shift ;; - -h|--help) + -h | --help) echo "${USAGE}" exit 0 ;; - -i|--install-hooks) + -i | --install-hooks) INSTALL_HOOKS=1 shift ;; @@ -153,6 +154,9 @@ for req_file in "requirements-dev.txt" "requirements-test.txt" "requirements.txt fi done +# Install all necessary mypy type stubs +mypy --install-types src/ + # Install git pre-commit hooks now or later. pre-commit install ${INSTALL_HOOKS:+"--install-hooks"} @@ -160,7 +164,8 @@ pre-commit install ${INSTALL_HOOKS:+"--install-hooks"} # This could fail if the remotes are already setup, but that is ok. set +o errexit -eval "$(python3 << 'END_OF_LINE' +eval "$( + python3 << 'END_OF_LINE' from pathlib import Path import yaml import sys