diff --git a/.ansible-lint b/.ansible-lint new file mode 100644 index 0000000..0e80b05 --- /dev/null +++ b/.ansible-lint @@ -0,0 +1,22 @@ +--- +# See https://ansible-lint.readthedocs.io/en/latest/configuring.html +# for a list of the configuration elements that can exist in this +# file. +enable_list: + # Useful checks that one must opt-into. See here for more details: + # https://ansible-lint.readthedocs.io/en/latest/rules.html + - fcqn-builtins + - no-log-password + - no-same-owner +exclude_paths: + # This exclusion is implicit, unless exclude_paths is defined + - .cache + # Seems wise to ignore this too + - .github +kinds: + # This will force our systemd specific molecule configurations to be treated + # as plain yaml files by ansible-lint. This mirrors the default kind + # configuration in ansible-lint for molecule configurations: + # yaml: "**/molecule/*/{base,molecule}.{yaml,yml}" + - yaml: "**/molecule/*/molecule-{no,with}-systemd.yml" +use_default_rules: true diff --git a/.bandit.yml b/.bandit.yml index 7b89269..0b53a96 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -11,4 +11,4 @@ tests: # - B102 skips: - - B101 # skip "assert used" check since assertions are required in pytests + - B101 # skip "assert used" check since assertions are required in pytests diff --git a/.flake8 b/.flake8 index 8f00fc9..92ff826 100644 --- a/.flake8 +++ b/.flake8 @@ -3,6 +3,8 @@ max-line-length = 80 # Select (turn on) # * Complexity violations reported by mccabe (C) - # http://flake8.pycqa.org/en/latest/user/error-codes.html#error-violation-codes +# * Documentation conventions compliance reported by pydocstyle (D) - +# http://www.pydocstyle.org/en/stable/error_codes.html # * Default errors and warnings reported by pycodestyle (E and W) - # https://pycodestyle.readthedocs.io/en/latest/intro.html#error-codes # * Default errors reported by pyflakes (F) - diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..371258c --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,10 @@ +# Each line is a file pattern followed by one or more owners. + +# These owners will be the default owners for everything in the +# repo. Unless a later match takes precedence, these owners will be +# requested for review when someone opens a pull request. +* @dav3r @felddy @jsf9k @mcdonnnj + +# These folks own any files in the .github directory at the root of +# the repository and any of its subdirectories. +/.github/ @dav3r @felddy @jsf9k @mcdonnnj diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..a3bcd94 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,18 @@ +--- + +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + + - package-ecosystem: "terraform" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/lineage.yml b/.github/lineage.yml new file mode 100644 index 0000000..df04737 --- /dev/null +++ b/.github/lineage.yml @@ -0,0 +1,5 @@ +--- +lineage: + skeleton: + remote-url: https://github.com/cisagov/skeleton-python-library.git +version: '1' diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..31d1120 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,240 @@ +--- +name: build + +on: + push: + pull_request: + repository_dispatch: + types: [apb] + +env: + CURL_CACHE_DIR: ~/.cache/curl + PIP_CACHE_DIR: ~/.cache/pip + PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit + RUN_TMATE: ${{ secrets.RUN_TMATE }} + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - id: setup-env + uses: cisagov/setup-env-github-action@develop + - uses: actions/checkout@v2 + - id: setup-python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + # We need the Go version and Go cache location for the actions/cache step, + # so the Go installation must happen before that. + - uses: actions/setup-go@v2 + with: + go-version: '1.16' + - name: Store installed Go version + id: go-version + run: | + echo "::set-output name=version::"\ + "$(go version | sed 's/^go version go\([0-9.]\+\) .*/\1/')" + - name: Lookup Go cache directory + id: go-cache + run: | + echo "::set-output name=dir::$(go env GOCACHE)" + - uses: actions/cache@v2 + env: + BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ + py${{ steps.setup-python.outputs.python-version }}-\ + go${{ steps.go-version.outputs.version }}-\ + packer${{ steps.setup-env.outputs.packer-version }}-\ + tf${{ steps.setup-env.outputs.terraform-version }}-" + with: + # Note that the .terraform directory IS NOT included in the + # cache because if we were caching, then we would need to use + # the `-upgrade=true` option. This option blindly pulls down the + # latest modules and providers instead of checking to see if an + # update is required. That behavior defeats the benefits of caching. + # so there is no point in doing it for the .terraform directory. + path: | + ${{ env.PIP_CACHE_DIR }} + ${{ env.PRE_COMMIT_CACHE_DIR }} + ${{ env.CURL_CACHE_DIR }} + ${{ steps.go-cache.outputs.dir }} + # We do not use '**/setup.py' in the cache key so only the 'setup.py' + # file in the root of the repository is used. This is in case a Python + # package were to have a 'setup.py' as part of its internal codebase. + key: "${{ env.BASE_CACHE_KEY }}\ + ${{ hashFiles('**/requirements-test.txt') }}-\ + ${{ hashFiles('**/requirements.txt') }}-\ + ${{ hashFiles('**/.pre-commit-config.yaml') }}-\ + ${{ hashFiles('setup.py') }}" + restore-keys: | + ${{ env.BASE_CACHE_KEY }} + - name: Setup curl cache + run: mkdir -p ${{ env.CURL_CACHE_DIR }} + - name: Install Packer + env: + PACKER_VERSION: ${{ steps.setup-env.outputs.packer-version }} + run: | + PACKER_ZIP="packer_${PACKER_VERSION}_linux_amd64.zip" + curl --output ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ + --time-cond ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ + --location \ + "https://releases.hashicorp.com/packer/${PACKER_VERSION}/${PACKER_ZIP}" + sudo unzip -d /opt/packer \ + ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" + sudo mv /usr/local/bin/packer /usr/local/bin/packer-default + sudo ln -s /opt/packer/packer /usr/local/bin/packer + - uses: hashicorp/setup-terraform@v1 + with: + terraform_version: ${{ steps.setup-env.outputs.terraform-version }} + - name: Install shfmt + env: + PACKAGE_URL: mvdan.cc/sh/v3/cmd/shfmt + PACKAGE_VERSION: ${{ steps.setup-env.outputs.shfmt-version }} + run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} + - name: Install Terraform-docs + env: + PACKAGE_URL: github.com/terraform-docs/terraform-docs + PACKAGE_VERSION: ${{ steps.setup-env.outputs.terraform-docs-version }} + run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade --requirement requirements-test.txt + - name: Set up pre-commit hook environments + run: pre-commit install-hooks + - name: Run pre-commit on all files + run: pre-commit run --all-files + - name: Setup tmate debug session + uses: mxschmitt/action-tmate@v3 + if: env.RUN_TMATE + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: + - "3.6" + - "3.7" + - "3.8" + - "3.9" + - "3.10" + steps: + - uses: actions/checkout@v2 + - id: setup-python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/cache@v2 + env: + BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ + py${{ steps.setup-python.outputs.python-version }}-" + with: + path: ${{ env.PIP_CACHE_DIR }} + # We do not use '**/setup.py' in the cache key so only the 'setup.py' + # file in the root of the repository is used. This is in case a Python + # package were to have a 'setup.py' as part of its internal codebase. + key: "${{ env.BASE_CACHE_KEY }}\ + ${{ hashFiles('**/requirements-test.txt') }}-\ + ${{ hashFiles('**/requirements.txt') }}-\ + ${{ hashFiles('setup.py') }}" + restore-keys: | + ${{ env.BASE_CACHE_KEY }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade --requirement requirements-test.txt + - name: Run tests + env: + RELEASE_TAG: ${{ github.event.release.tag_name }} + run: pytest + - name: Upload coverage report + run: coveralls + env: + COVERALLS_FLAG_NAME: "py${{ matrix.python-version }}" + COVERALLS_PARALLEL: true + COVERALLS_SERVICE_NAME: github + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + if: success() + - name: Setup tmate debug session + uses: mxschmitt/action-tmate@v3 + if: env.RUN_TMATE + coveralls-finish: + runs-on: ubuntu-latest + needs: test + steps: + - uses: actions/checkout@v2 + - id: setup-python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - uses: actions/cache@v2 + env: + BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ + py${{ steps.setup-python.outputs.python-version }}-" + with: + path: ${{ env.PIP_CACHE_DIR }} + # We do not use '**/setup.py' in the cache key so only the 'setup.py' + # file in the root of the repository is used. This is in case a Python + # package were to have a 'setup.py' as part of its internal codebase. + key: "${{ env.BASE_CACHE_KEY }}\ + ${{ hashFiles('**/requirements-test.txt') }}-\ + ${{ hashFiles('**/requirements.txt') }}-\ + ${{ hashFiles('setup.py') }}" + restore-keys: | + ${{ env.BASE_CACHE_KEY }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade --requirement requirements-test.txt + - name: Finished coveralls reports + run: coveralls --finish + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Setup tmate debug session + uses: mxschmitt/action-tmate@v3 + if: env.RUN_TMATE + build: + runs-on: ubuntu-latest + needs: [lint, test] + strategy: + fail-fast: false + matrix: + python-version: + - "3.6" + - "3.7" + - "3.8" + - "3.9" + - "3.10" + steps: + - uses: actions/checkout@v2 + - id: setup-python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/cache@v2 + env: + BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ + py${{ steps.setup-python.outputs.python-version }}-" + with: + path: ${{ env.PIP_CACHE_DIR }} + # We do not use '**/setup.py' in the cache key so only the 'setup.py' + # file in the root of the repository is used. This is in case a Python + # package were to have a 'setup.py' as part of its internal codebase. + key: "${{ env.BASE_CACHE_KEY }}\ + ${{ hashFiles('**/requirements.txt') }}-\ + ${{ hashFiles('setup.py') }}" + restore-keys: | + ${{ env.BASE_CACHE_KEY }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip wheel + pip install --upgrade --requirement requirements.txt + - name: Build artifacts + run: python3 setup.py sdist bdist_wheel + - name: Upload artifacts + uses: actions/upload-artifact@v2 + with: + name: dist-${{ matrix.python-version }} + path: dist + - name: Setup tmate debug session + uses: mxschmitt/action-tmate@v3 + if: env.RUN_TMATE diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..b880c44 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,68 @@ +--- +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +name: "CodeQL" + +on: + push: + # Dependabot triggered push events have read-only access, but uploading code + # scanning requires write access. + branches-ignore: [dependabot/**] + pull_request: + # The branches below must be a subset of the branches above + branches: [develop] + schedule: + - cron: '0 14 * * 6' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + # Override automatic language detection by changing the below list + # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', + # 'python'] + language: ['python'] + # Learn more... + # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a + # config file. By default, queries listed here will override any + # specified in a config file. Prefix the list here with "+" to use + # these queries and those in the config file. queries: + # ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or + # Java). If this step fails, then you should remove it and run the build + # manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following + # three lines and modify them (or add more) to build your code if your + # project uses a compiled language + + # - run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/.gitignore b/.gitignore index 724760e..53e6544 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,11 @@ -*.egg-info +# This file specifies intentionally untracked files that Git should ignore. +# Files already tracked by Git are not affected. +# See: https://git-scm.com/docs/gitignore + +## Python ## __pycache__ -.python-version .coverage +.mypy_cache .pytest_cache +.python-version +*.egg-info diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 0000000..46d45f3 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,10 @@ +[settings] +combine_star=true +force_sort_within_sections=true + +import_heading_stdlib=Standard Python Libraries +import_heading_thirdparty=Third-Party Libraries +import_heading_firstparty=cisagov Libraries + +# Run isort under the black profile to align with our other Python linting +profile=black diff --git a/.mdl_config.yaml b/.mdl_config.yaml new file mode 100644 index 0000000..b36f943 --- /dev/null +++ b/.mdl_config.yaml @@ -0,0 +1,50 @@ +--- + +# Default state for all rules +default: true + +# MD003/heading-style/header-style - Heading style +MD003: + # Enforce the ATX-closed style of header + style: "atx_closed" + +# MD004/ul-style - Unordered list style +MD004: + # Enforce dashes for unordered lists + style: "dash" + +# MD013/line-length - Line length +MD013: + # Do not enforce for code blocks + code_blocks: false + # Do not enforce for tables + tables: false + +# MD024/no-duplicate-heading/no-duplicate-header - Multiple headings with the +# same content +MD024: + # Allow headers with the same content as long as they are not in the same + # parent heading + allow_different_nesting: true + +# MD029/ol-prefix - Ordered list item prefix +MD029: + # Enforce the `1.` style for ordered lists + style: "one" + +# MD033/no-inline-html - Inline HTML +MD033: + # The h1 and img elements are allowed to permit header images + allowed_elements: + - h1 + - img + +# MD035/hr-style - Horizontal rule style +MD035: + # Enforce dashes for horizontal rules + style: "---" + +# MD046/code-block-style Code block style +MD046: + # Enforce the fenced style for code blocks + style: "fenced" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a672df8..f9415c1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,11 +1,17 @@ --- +default_language_version: + # force all unspecified python hooks to run python3 + python: python3 + repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.2.1 + rev: v4.1.0 hooks: + - id: check-case-conflict - id: check-executables-have-shebangs - id: check-json - id: check-merge-conflict + - id: check-toml - id: check-xml - id: debug-statements - id: detect-aws-credentials @@ -22,35 +28,55 @@ repos: - --autofix - id: requirements-txt-fixer - id: trailing-whitespace + + # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.15.0 + rev: v0.30.0 hooks: - id: markdownlint - # The LICENSE.md must match the license text exactly for - # GitHub's autorecognition fu to work, so we should leave it - # alone. - exclude: LICENSE.md + args: + - --config=.mdl_config.yaml + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v2.5.1 + hooks: + - id: prettier - repo: https://github.com/adrienverge/yamllint - rev: v1.15.0 + rev: v1.26.3 hooks: - id: yamllint - - repo: https://github.com/detailyang/pre-commit-shell - rev: 1.0.4 + args: + - --strict + + # pre-commit hooks + - repo: https://github.com/pre-commit/pre-commit + rev: v2.16.0 hooks: - - id: shell-lint - - repo: https://gitlab.com/pycqa/flake8 - rev: 3.7.7 + - id: validate_manifest + + # Shell script hooks + - repo: https://github.com/cisagov/pre-commit-shfmt + rev: v0.0.2 hooks: - - id: flake8 - additional_dependencies: - - flake8-docstrings - - repo: https://github.com/asottile/pyupgrade - rev: v1.16.3 + - id: shfmt + args: + # Indent by two spaces + - -i + - '2' + # Binary operators may start a line + - -bn + # Switch cases are indented + - -ci + # Redirect operators are followed by a space + - -sr + - repo: https://github.com/detailyang/pre-commit-shell + rev: 1.0.5 hooks: - - id: pyupgrade + - id: shell-lint + + # Python hooks # Run bandit on "tests" tree with a configuration - repo: https://github.com/PyCQA/bandit - rev: 2a1dbab + rev: 1.7.1 hooks: - id: bandit name: bandit (tests tree) @@ -59,31 +85,57 @@ repos: - --config=.bandit.yml # Run bandit everything but tests directory - repo: https://github.com/PyCQA/bandit - rev: 2a1dbab + rev: 1.7.0 hooks: - id: bandit name: bandit (everything else) exclude: tests - - repo: https://github.com/python/black - rev: 19.3b0 + - repo: https://github.com/psf/black + rev: 21.12b0 hooks: - id: black - - repo: https://github.com/ansible/ansible-lint.git - rev: v4.1.0a0 + - repo: https://gitlab.com/pycqa/flake8 + rev: 3.9.2 + hooks: + - id: flake8 + additional_dependencies: + - flake8-docstrings + - repo: https://github.com/PyCQA/isort + rev: 5.10.1 + hooks: + - id: isort + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.931 + hooks: + - id: mypy + - repo: https://github.com/asottile/pyupgrade + rev: v2.31.0 + hooks: + - id: pyupgrade + + # Ansible hooks + - repo: https://github.com/ansible-community/ansible-lint + rev: v5.3.2 hooks: - id: ansible-lint - # files: molecule/default/playbook.yml - - repo: https://github.com/antonbabenko/pre-commit-terraform.git - rev: v1.11.0 + # files: molecule/default/playbook.yml + + # Terraform hooks + - repo: https://github.com/antonbabenko/pre-commit-terraform + rev: v1.62.3 hooks: - id: terraform_fmt - - id: terraform_validate_no_variables - - id: terraform_docs + - id: terraform_validate + + # Docker hooks - repo: https://github.com/IamTheFij/docker-pre-commit - rev: v1.0.0 + rev: v2.0.1 hooks: - id: docker-compose-check - - repo: https://github.com/prettier/prettier - rev: 1.17.0 + + # Packer hooks + - repo: https://github.com/cisagov/pre-commit-packer + rev: v0.0.2 hooks: - - id: prettier + - id: packer_validate + - id: packer_fmt diff --git a/.prettierignore b/.prettierignore index f1e840c..738d402 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,3 +1,5 @@ +# Already being linted by pretty-format-json +*.json # Already being linted by mdl *.md # Already being linted by yamllint diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 88bd1a7..0000000 --- a/.travis.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -dist: xenial -language: python -python: 3.7 -services: docker - -install: - - pip install --upgrade -r requirements-test.txt -script: - - pre-commit run --all-files - - pytest -v --cov=lcgit -after_success: - - coveralls diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 320324f..303725e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,8 +8,8 @@ of contribution, and don't want a wall of rules to get in the way of that. Before contributing, we encourage you to read our CONTRIBUTING policy -(you are here), our [LICENSE](LICENSE.md), and our -[README](README.md), all of which should be in this repository. +(you are here), our [LICENSE](LICENSE), and our [README](README.md), +all of which should be in this repository. ## Issues ## @@ -46,20 +46,77 @@ There are a few ways to do this, but we prefer to use create and manage a Python virtual environment specific to this project. +If you already have `pyenv` and `pyenv-virtualenv` configured you can +take advantage of the `setup-env` tool in this repo to automate the +entire environment configuration process. + +```console +./setup-env +``` + +Otherwise, follow the steps below to manually configure your +environment. + #### Installing and using `pyenv` and `pyenv-virtualenv` #### -On the Mac, installation is as simple as `brew install pyenv -pyenv-virtualenv` and adding this to your profile: +On the Mac, we recommend installing [brew](https://brew.sh/). Then +installation is as simple as `brew install pyenv pyenv-virtualenv` and +adding this to your profile: ```bash +export PYENV_ROOT="$HOME/.pyenv" +export PATH="$PYENV_ROOT/bin:$PATH" +eval "$(pyenv init --path)" eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" ``` -For Linux (or on the Mac, if you don't want to use `brew`) you can use +For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you +don't want to use `brew`) you can use [pyenv/pyenv-installer](https://github.com/pyenv/pyenv-installer) to -install the necessary tools. When you are finished you will need to -add the same two lines above to your profile. +install the necessary tools. Before running this ensure that you have +installed the prerequisites for your platform according to the +[`pyenv` wiki +page](https://github.com/pyenv/pyenv/wiki/common-build-problems). + +On WSL you should treat your platform as whatever Linux distribution +you've chosen to install. + +Once you have installed `pyenv` you will need to add the following +lines to your `.bash_profile` (or `.profile`): + +```bash +export PYENV_ROOT="$HOME/.pyenv" +export PATH="$PYENV_ROOT/bin:$PATH" +eval "$(pyenv init --path)" +``` + +and then add the following lines to your `.bashrc`: + +```bash +eval "$(pyenv init -)" +eval "$(pyenv virtualenv-init -)" +``` + +If you want more information about setting up `pyenv` once installed, please run + +```console +pyenv init +``` + +and + +```console +pyenv virtualenv-init +``` + +for the current configuration instructions. + +If you are using a shell other than `bash` you should follow the +instructions that the `pyenv-installer` script outputs. + +You will need to reload your shell for these changes to take effect so +you can begin to use `pyenv`. For a list of Python versions that are already installed and ready to use with `pyenv`, use the command `pyenv versions`. To see a list of @@ -77,18 +134,18 @@ Once `pyenv` and `pyenv-virtualenv` are installed on your system, you can create and configure the Python virtual environment with these commands: -```bash +```console cd lcgit pyenv virtualenv lcgit pyenv local lcgit -pip install -r requirements-dev.txt +pip install --requirement requirements-dev.txt ``` #### Installing the pre-commit hook #### Now setting up pre-commit is as simple as: -```bash +```console pre-commit install ``` diff --git a/LICENSE.md b/LICENSE similarity index 100% rename from LICENSE.md rename to LICENSE diff --git a/README.md b/README.md index da6efce..94aaf6a 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,10 @@ -# lcgit 🎰 +# lcgit 🎰 # -[![Build Status](https://travis-ci.com/cisagov/lcgit.svg?branch=develop)](https://travis-ci.com/cisagov/lcgit) +[![GitHub Build Status](https://github.com/cisagov/lcgit/workflows/build/badge.svg)](https://github.com/cisagov/lcgit/actions) [![Coverage Status](https://coveralls.io/repos/github/cisagov/lcgit/badge.svg?branch=develop)](https://coveralls.io/github/cisagov/lcgit?branch=develop) [![Total alerts](https://img.shields.io/lgtm/alerts/g/cisagov/lcgit.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/lcgit/alerts/) [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/cisagov/lcgit.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/lcgit/context:python) +[![Known Vulnerabilities](https://snyk.io/test/github/cisagov/lcgit/develop/badge.svg)](https://snyk.io/test/github/cisagov/lcgit) Do you want to loop randomly through every item in huge sequence without outputting the same item twice? Would you like to do this while keeping @@ -67,16 +68,16 @@ for i in lcg(range(100_000_000_000_000)): print(i) ``` -## NOOICE! đŸ•ē +## NOOICE! đŸ•ē ## -## Contributing +## Contributing ## -We welcome contributions! Please see [here](CONTRIBUTING.md) for +We welcome contributions! Please see [`CONTRIBUTING.md`](CONTRIBUTING.md) for details. -## License +## License ## -This project is in the worldwide [public domain](LICENSE.md). +This project is in the worldwide [public domain](LICENSE). This project is in the public domain within the United States, and copyright and related rights in the work worldwide are waived through diff --git a/bump_version.sh b/bump_version.sh new file mode 100755 index 0000000..704b162 --- /dev/null +++ b/bump_version.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +# bump_version.sh (show|major|minor|patch|prerelease|build) + +set -o nounset +set -o errexit +set -o pipefail + +VERSION_FILE=src/lcgit/_version.py + +HELP_INFORMATION="bump_version.sh (show|major|minor|patch|prerelease|build|finalize)" + +old_version=$(sed -n "s/^__version__ = \"\(.*\)\"$/\1/p" $VERSION_FILE) + +if [ $# -ne 1 ]; then + echo "$HELP_INFORMATION" +else + case $1 in + major | minor | patch | prerelease | build) + new_version=$(python -c "import semver; print(semver.bump_$1('$old_version'))") + echo Changing version from "$old_version" to "$new_version" + # A temp file is used to provide compatability with macOS development + # as a result of macOS using the BSD version of sed + tmp_file=/tmp/version.$$ + sed "s/$old_version/$new_version/" $VERSION_FILE > $tmp_file + mv $tmp_file $VERSION_FILE + git add $VERSION_FILE + git commit -m"Bump version from $old_version to $new_version" + git push + ;; + finalize) + new_version=$(python -c "import semver; print(semver.finalize_version('$old_version'))") + echo Changing version from "$old_version" to "$new_version" + # A temp file is used to provide compatability with macOS development + # as a result of macOS using the BSD version of sed + tmp_file=/tmp/version.$$ + sed "s/$old_version/$new_version/" $VERSION_FILE > $tmp_file + mv $tmp_file $VERSION_FILE + git add $VERSION_FILE + git commit -m"Bump version from $old_version to $new_version" + git push + ;; + show) + echo "$old_version" + ;; + *) + echo "$HELP_INFORMATION" + ;; + esac +fi diff --git a/requirements-dev.txt b/requirements-dev.txt index f122cc5..1d7e302 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,4 @@ --r requirements-test.txt +--requirement requirements-test.txt ipython +mypy +semver diff --git a/requirements-test.txt b/requirements-test.txt index a4af5aa..1d0a5a4 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1 +1,2 @@ --e .[test] +--editable .[test] +--requirement requirements.txt diff --git a/requirements.txt b/requirements.txt index d6e1198..8b75fe9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,3 @@ --e . +# Note: Add any additional requirements to setup.py's install_requires field +--editable . +wheel diff --git a/setup-env b/setup-env new file mode 100755 index 0000000..3392f60 --- /dev/null +++ b/setup-env @@ -0,0 +1,193 @@ +#!/usr/bin/env bash + +set -o nounset +set -o errexit +set -o pipefail + +USAGE=$( + cat << 'END_OF_LINE' +Configure a development environment for this repository. + +It does the following: + - Verifies pyenv and pyenv-virtualenv are installed. + - Creates a Python virtual environment. + - Configures the activation of the virtual enviroment for the repo directory. + - Installs the requirements needed for development (including mypy type stubs). + - Installs git pre-commit hooks. + - Configures git upstream remote "lineage" repositories. + +Usage: + setup-env [options] [virt_env_name] + setup-env (-h | --help) + +Options: + -f --force Delete virtual enviroment if it already exists. + -h --help Show this message. + -i --install-hooks Install hook environments for all environments in the + pre-commit config file. + +END_OF_LINE +) + +# Flag to force deletion and creation of virtual environment +FORCE=0 + +# Positional parameters +PARAMS="" + +# Parse command line arguments +while (("$#")); do + case "$1" in + -f | --force) + FORCE=1 + shift + ;; + -h | --help) + echo "${USAGE}" + exit 0 + ;; + -i | --install-hooks) + INSTALL_HOOKS=1 + shift + ;; + -*) # unsupported flags + echo "Error: Unsupported flag $1" >&2 + exit 1 + ;; + *) # preserve positional arguments + PARAMS="$PARAMS $1" + shift + ;; + esac +done + +# set positional arguments in their proper place +eval set -- "$PARAMS" + +# Check to see if pyenv is installed +if [ -z "$(command -v pyenv)" ] || [ -z "$(command -v pyenv-virtualenv)" ]; then + echo "pyenv and pyenv-virtualenv are required." + if [[ "$OSTYPE" == "darwin"* ]]; then + cat << 'END_OF_LINE' + + On the Mac, we recommend installing brew, https://brew.sh/. Then installation + is as simple as `brew install pyenv pyenv-virtualenv` and adding this to your + profile: + + eval "$(pyenv init -)" + eval "$(pyenv virtualenv-init -)" + +END_OF_LINE + + fi + cat << 'END_OF_LINE' + For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you don't want + to use "brew") you can use https://github.com/pyenv/pyenv-installer to install + the necessary tools. Before running this ensure that you have installed the + prerequisites for your platform according to the pyenv wiki page, + https://github.com/pyenv/pyenv/wiki/common-build-problems. + + On WSL you should treat your platform as whatever Linux distribution you've + chosen to install. + + Once you have installed "pyenv" you will need to add the following lines to + your ".bashrc": + + export PATH="$PATH:$HOME/.pyenv/bin" + eval "$(pyenv init -)" + eval "$(pyenv virtualenv-init -)" +END_OF_LINE + exit 1 +fi + +set +o nounset +# Determine the virtual environment name +if [ "$1" ]; then + # Use the user-provided environment name + env_name=$1 +else + # Set the environment name to the last part of the working directory. + env_name=${PWD##*/} +fi +set -o nounset + +# Remove any lingering local configuration. +if [ $FORCE -ne 0 ]; then + rm -f .python-version + pyenv virtualenv-delete --force "${env_name}" || true +elif [[ -f .python-version ]]; then + cat << 'END_OF_LINE' + An existing .python-version file was found. Either remove this file yourself + or re-run with --force option to have it deleted along with the associated + virtual environment. + + rm .python-version + +END_OF_LINE + exit 1 +fi + +# Create a new virtual environment for this project +if ! pyenv virtualenv "${env_name}"; then + cat << END_OF_LINE + An existing virtual environment named $env_name was found. Either delete this + environment yourself or re-run with --force option to have it deleted. + + pyenv virtualenv-delete ${env_name} + +END_OF_LINE + exit 1 +fi + +# Set the local application-specific Python version(s) by writing the +# version name to a file named `.python-version'. +pyenv local "${env_name}" + +# Upgrade pip and friends +python3 -m pip install --upgrade pip setuptools wheel + +# Find a requirements file (if possible) and install +for req_file in "requirements-dev.txt" "requirements-test.txt" "requirements.txt"; do + if [[ -f $req_file ]]; then + pip install --requirement $req_file + break + fi +done + +# Install all necessary mypy type stubs +mypy --install-types src/ + +# Install git pre-commit hooks now or later. +pre-commit install ${INSTALL_HOOKS:+"--install-hooks"} + +# Setup git remotes from lineage configuration +# This could fail if the remotes are already setup, but that is ok. +set +o errexit + +eval "$( + python3 << 'END_OF_LINE' +from pathlib import Path +import yaml +import sys + +LINEAGE_CONFIG = Path(".github/lineage.yml") + +if not LINEAGE_CONFIG.exists(): + print("No lineage configuration found.", file=sys.stderr) + sys.exit(0) + +with LINEAGE_CONFIG.open("r") as f: + lineage = yaml.safe_load(stream=f) + +if lineage["version"] == "1": + for parent_name, v in lineage["lineage"].items(): + remote_url = v["remote-url"] + print(f"git remote add {parent_name} {remote_url};") + print(f"git remote set-url --push {parent_name} no_push;") +else: + print(f'Unsupported lineage version: {lineage["version"]}', file=sys.stderr) +END_OF_LINE +)" + +# Qapla +echo "Success!" diff --git a/setup.py b/setup.py index 770abb7..6182db9 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ """ -setup module for lcg. +This is the setup module for the lcgit project. Based on: @@ -8,40 +8,57 @@ - https://blog.ionelmc.ro/2014/05/25/python-packaging/#the-structure """ +# Standard Python Libraries +import codecs from glob import glob -from os.path import splitext, basename +from os.path import abspath, basename, dirname, join, splitext -from setuptools import setup, find_packages +# Third-Party Libraries +from setuptools import find_packages, setup def readme(): """Read in and return the contents of the project's README.md file.""" - with open("README.md") as f: + with open("README.md", encoding="utf-8") as f: return f.read() -def package_vars(version_file): - """Read in and return the variables defined by the version_file.""" - pkg_vars = {} - with open(version_file) as f: - exec(f.read(), pkg_vars) # nosec - return pkg_vars +# Below two methods were pulled from: +# https://packaging.python.org/guides/single-sourcing-package-version/ +def read(rel_path): + """Open a file for reading from a given relative path.""" + here = abspath(dirname(__file__)) + with codecs.open(join(here, rel_path), "r") as fp: + return fp.read() + + +def get_version(version_file): + """Extract a version number from the given file path.""" + for line in read(version_file).splitlines(): + if line.startswith("__version__"): + delim = '"' if '"' in line else "'" + return line.split(delim)[1] + raise RuntimeError("Unable to find version string.") setup( name="lcgit", # Versions should comply with PEP440 - version=package_vars("src/lcgit/_version.py")["__version__"], - description="LCG iterator python library", + version=get_version("src/lcgit/_version.py"), + description="LCG iterator Python library", long_description=readme(), long_description_content_type="text/markdown", - # NCATS "homepage" - url="https://www.us-cert.gov/resources/ncats", - # The project's main homepage - download_url="https://github.com/cisagov/lcgit", + # Landing page for CISA's cybersecurity mission + url="https://www.cisa.gov/cybersecurity", + # Additional URLs for this project per + # https://packaging.python.org/guides/distributing-packages-using-setuptools/#project-urls + project_urls={ + "Source": "https://github.com/cisagov/lcgit", + "Tracker": "https://github.com/cisagov/lcgit/issues", + }, # Author details - author="Cyber and Infrastructure Security Agency", - author_email="ncats@hq.dhs.gov", + author="Cybersecurity and Infrastructure Security Agency", + author_email="github@cisa.dhs.gov", license="License :: CC0 1.0 Universal (CC0 1.0) Public Domain Dedication", # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ @@ -59,7 +76,10 @@ def package_vars(version_file): "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", ], + python_requires=">=3.6", # What does your project relate to? keywords=["iterator", "lcg", "network", "random"], packages=find_packages(where="src"), @@ -67,5 +87,19 @@ def package_vars(version_file): py_modules=[splitext(basename(path))[0] for path in glob("src/*.py")], include_package_data=True, install_requires=[], - extras_require={"test": ["pre-commit", "pytest", "pytest-cov", "coveralls"]}, + extras_require={ + "test": [ + "coverage", + # coveralls 1.11.0 added a service number for calls from + # GitHub Actions. This caused a regression which resulted in a 422 + # response from the coveralls API with the message: + # Unprocessable Entity for url: https://coveralls.io/api/v1/jobs + # 1.11.1 fixed this issue, but to ensure expected behavior we'll pin + # to never grab the regression version. + "coveralls != 1.11.0", + "pre-commit", + "pytest-cov", + "pytest", + ] + }, ) diff --git a/src/lcgit/__init__.py b/src/lcgit/__init__.py index bcb9f8e..d89314c 100644 --- a/src/lcgit/__init__.py +++ b/src/lcgit/__init__.py @@ -1,9 +1,13 @@ """A fast generator of random, non-repeating, maximal length sequences. -This library is used to generate randomized sequences from python sequences +This library is used to generate randomized sequences from Python sequences and IP networks. """ -from .lcgit import lcg +# We disable a Flake8 check for "Module imported but unused (F401)" here because +# although this import is not directly used, it populates the value +# package_name.__version__, which is used to get version information about this +# Python package. from ._version import __version__ # noqa: F401 +from .lcgit import lcg __all__ = ["lcg"] diff --git a/src/lcgit/lcgit.py b/src/lcgit/lcgit.py index db97f74..346870b 100755 --- a/src/lcgit/lcgit.py +++ b/src/lcgit/lcgit.py @@ -9,10 +9,11 @@ see: https://en.wikipedia.org/wiki/Linear_congruential_generator """ -from random import randint, shuffle -from math import sin +# Standard Python Libraries from collections.abc import Sequence from ipaddress import _BaseNetwork +from math import sin +from random import randint, shuffle def _lcg_params(u, v): diff --git a/tag.sh b/tag.sh new file mode 100755 index 0000000..e1f7447 --- /dev/null +++ b/tag.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -o nounset +set -o errexit +set -o pipefail + +version=$(./bump_version.sh show) + +git tag "v$version" && git push --tags diff --git a/tests/conftest.py b/tests/conftest.py index b060645..39e2be2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,7 @@ https://docs.pytest.org/en/latest/writing_plugins.html#conftest-py-plugins """ +# Third-Party Libraries import pytest @@ -12,6 +13,11 @@ def pytest_addoption(parser): ) +def pytest_configure(config): + """Register new markers.""" + config.addinivalue_line("markers", "slow: mark test as slow") + + def pytest_collection_modifyitems(config, items): """Modify collected tests based on custom marks and commandline options.""" if config.getoption("--runslow"): diff --git a/tests/lcgit_test.py b/tests/test_lcgit.py similarity index 94% rename from tests/lcgit_test.py rename to tests/test_lcgit.py index 0736f7c..4b077dd 100644 --- a/tests/lcgit_test.py +++ b/tests/test_lcgit.py @@ -1,8 +1,13 @@ #!/usr/bin/env pytest -vs """Tests for lcgit.""" -import pytest +# Standard Python Libraries from ipaddress import ip_network as net + +# Third-Party Libraries +import pytest + +# cisagov Libraries from lcgit import lcg, lcgit sequences = [ @@ -25,7 +30,7 @@ @pytest.mark.parametrize("sequence", sequences) def test_counts_and_dups(sequence): """Verify LCG output integrity.""" - answer = sorted([i for i in sequence]) + answer = sorted(i for i in sequence) my_lcg = lcg(sequence) accumulated = [] count = 0 @@ -40,7 +45,7 @@ def test_counts_and_dups(sequence): @pytest.mark.parametrize("sequence", sequences) def test_state_save_and_restore(sequence): """Verify state save and restore.""" - answer = sorted([i for i in sequence]) + answer = sorted(i for i in sequence) lcg1 = lcg(sequence, emit=True) accumulated = [] break_at = len(lcg1) / 2