From 8e0b1d73868912f6736ae76af100b2d24b4f53b6 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 3 May 2019 14:30:56 -0400 Subject: [PATCH 001/282] Add pydoc warnings to flake8 config Also add flake8-docstrings as an additional dependency in the .pre-commit-config.yml file. --- .flake8 | 4 +++- .pre-commit-config.yaml | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index d937d88..92ff826 100644 --- a/.flake8 +++ b/.flake8 @@ -3,6 +3,8 @@ max-line-length = 80 # Select (turn on) # * Complexity violations reported by mccabe (C) - # http://flake8.pycqa.org/en/latest/user/error-codes.html#error-violation-codes +# * Documentation conventions compliance reported by pydocstyle (D) - +# http://www.pydocstyle.org/en/stable/error_codes.html # * Default errors and warnings reported by pycodestyle (E and W) - # https://pycodestyle.readthedocs.io/en/latest/intro.html#error-codes # * Default errors reported by pyflakes (F) - @@ -11,7 +13,7 @@ max-line-length = 80 # https://github.com/PyCQA/flake8-bugbear#list-of-warnings # * The B950 flake8-bugbear opinionated warning - # https://github.com/PyCQA/flake8-bugbear#opinionated-warnings -select = C,E,F,W,B,B950 +select = C,D,E,F,W,B,B950 # Ignore flake8's default warning about maximum line length, which has # a hard stop at the configured value. Instead we use # flake8-bugbear's B950, which allows up to 10% overage. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 926a491..fb12c99 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,6 +42,8 @@ repos: rev: 3.7.7 hooks: - id: flake8 + additional_dependencies: + - flake8-docstrings - repo: https://github.com/asottile/pyupgrade rev: v1.14.0 hooks: From 4d34c763dd9708a4745676f78c5ec4795ba9f5f8 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 3 May 2019 14:41:16 -0400 Subject: [PATCH 002/282] Make flake8 happy now that the pydoc warnings and error messages are turned on. --- example/example.py | 5 +++-- setup.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/example/example.py b/example/example.py index e520440..3f9cbfe 100755 --- a/example/example.py +++ b/example/example.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -"""example is an example Python library and tool +"""example is an example Python library and tool. Usage: example [--log-level=LEVEL] @@ -20,7 +20,7 @@ def example(): - """A dummy function.""" + """Print some logging messages.""" logging.debug("This is a debug message") logging.info("This is an info message") logging.warning("This is a warning message") @@ -29,6 +29,7 @@ def example(): def main(): + """Set up logging and call the example function.""" args = docopt.docopt(__doc__, version="0.0.1") # Set up logging diff --git a/setup.py b/setup.py index 057da60..63419fb 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ """ -setup module for example +This is the setup module for the example project. Based on: @@ -11,6 +11,7 @@ def readme(): + """Read in and return the contents of the project's README.md file.""" with open("README.md") as f: return f.read() From 6a64a512e40ba6a3cc0e6788fc67207f3e77ebd2 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 3 May 2019 16:49:35 -0400 Subject: [PATCH 003/282] Add a configuration file for Bandit, and configure pre-commit to use it. --- .bandit.yml | 12 ++++++++++++ .pre-commit-config.yaml | 2 ++ 2 files changed, 14 insertions(+) create mode 100644 .bandit.yml diff --git a/.bandit.yml b/.bandit.yml new file mode 100644 index 0000000..f122c97 --- /dev/null +++ b/.bandit.yml @@ -0,0 +1,12 @@ +# Configuration file for the Bandit python security scanner +# https://bandit.readthedocs.io/en/latest/config.html + +# Test are first included by `tests`, and then excluded by `skips`. +# If `tests` is empty, all tests are are considered included. + +tests: + #- B101 + #- B102 + +skips: + #- B101 # skip "assert used" check since assertions are required in pytests diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fb12c99..2a03327 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -52,6 +52,8 @@ repos: rev: 2a1dbab hooks: - id: bandit + args: + - --config=.bandit.yml - repo: https://github.com/ambv/black rev: 19.3b0 hooks: From 1a98e0ed062cc55719893bbbeafc3df40510b42d Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 3 May 2019 16:51:21 -0400 Subject: [PATCH 004/282] Improve english --- .bandit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bandit.yml b/.bandit.yml index f122c97..46bab94 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -1,7 +1,7 @@ # Configuration file for the Bandit python security scanner # https://bandit.readthedocs.io/en/latest/config.html -# Test are first included by `tests`, and then excluded by `skips`. +# Tests are first included by `tests`, and then excluded by `skips`. # If `tests` is empty, all tests are are considered included. tests: From e13fb008917ccd9cb6db1fac378307c6bdbf2ab9 Mon Sep 17 00:00:00 2001 From: Felddy Date: Sun, 5 May 2019 00:00:54 -0400 Subject: [PATCH 005/282] Add pytest examples and configure travis to call --- .travis.yml | 1 + example/__init__.py | 5 +++++ example/example.py | 5 +++-- setup.py | 2 +- tests/test_example.py | 27 +++++++++++++++++++++++++++ 5 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 example/__init__.py create mode 100644 tests/test_example.py diff --git a/.travis.yml b/.travis.yml index a4493e4..64a0c3b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,3 +8,4 @@ install: - pip install --upgrade -r requirements-test.txt script: - pre-commit run --all-files + - pytest -v diff --git a/example/__init__.py b/example/__init__.py new file mode 100644 index 0000000..0630045 --- /dev/null +++ b/example/__init__.py @@ -0,0 +1,5 @@ +"""The example library.""" +from .example import example_div + +__all__ = ["example_div"] +__version__ = "0.0.1" diff --git a/example/example.py b/example/example.py index 3f9cbfe..6382d8c 100755 --- a/example/example.py +++ b/example/example.py @@ -19,13 +19,14 @@ import docopt -def example(): +def example_div(x, y): """Print some logging messages.""" logging.debug("This is a debug message") logging.info("This is an info message") logging.warning("This is a warning message") logging.error("This is an error message") logging.critical("This is a critical message") + return x / y def main(): @@ -45,7 +46,7 @@ def main(): ) return 1 - example() + print(example_div(8, 2)) # Stop logging and clean up logging.shutdown() diff --git a/setup.py b/setup.py index 63419fb..1a73188 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ def readme(): keywords="skeleton", packages=["example"], install_requires=["docopt"], - extras_require={"test": ["pre-commit"]}, + extras_require={"test": ["pre-commit", "pytest"]}, # Conveniently allows one to run the CLI tool as `example` entry_points={"console_scripts": ["example = example.example:main"]}, ) diff --git a/tests/test_example.py b/tests/test_example.py new file mode 100644 index 0000000..fe3bfb8 --- /dev/null +++ b/tests/test_example.py @@ -0,0 +1,27 @@ +#!/usr/bin/env pytest -vs +"""Tests for example.""" + +import pytest + +from example import example_div + +div_params = [ + (1, 1, 1), + (2, 2, 1), + (0, 1, 0), + (8, 2, 4), + pytest.param(0, 0, 0, marks=pytest.mark.xfail(raises=ZeroDivisionError)), +] + + +@pytest.mark.parametrize("dividend, divisor, quotient", div_params) +def test_division(dividend, divisor, quotient): + """Verify division results.""" + result = example_div(dividend, divisor) + assert result == quotient, "result should equal quotient" # nosec + + +def test_zero_division(): + """Verify that division by zero throws the correct exception.""" + with pytest.raises(ZeroDivisionError): + example_div(1, 0) From edf020a169fa6a866aff08cc003393322de7c25c Mon Sep 17 00:00:00 2001 From: Felddy Date: Sun, 5 May 2019 00:16:06 -0400 Subject: [PATCH 006/282] Ignore coverage output --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 06be862..3f73e7f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.egg-info __pycache__ .python-version +.coverage From 124379fe47745262195a0448ae1aec4892354a0d Mon Sep 17 00:00:00 2001 From: Felddy Date: Sun, 5 May 2019 00:16:44 -0400 Subject: [PATCH 007/282] Add coverage check to travis config --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 1a73188..58fa07d 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ def readme(): setup( - name="add", + name="example", # Versions should comply with PEP440 version="0.0.1", description="Example python library", @@ -52,7 +52,7 @@ def readme(): keywords="skeleton", packages=["example"], install_requires=["docopt"], - extras_require={"test": ["pre-commit", "pytest"]}, + extras_require={"test": ["pre-commit", "pytest", "pytest-cov"]}, # Conveniently allows one to run the CLI tool as `example` entry_points={"console_scripts": ["example = example.example:main"]}, ) From 69ba1e2f3e45f600f18f286159e141c3ca138e03 Mon Sep 17 00:00:00 2001 From: Felddy Date: Sun, 5 May 2019 00:33:01 -0400 Subject: [PATCH 008/282] Add coveralls --- .travis.yml | 4 +++- setup.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 64a0c3b..0fa9cfa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,4 +8,6 @@ install: - pip install --upgrade -r requirements-test.txt script: - pre-commit run --all-files - - pytest -v + - pytest -v --cov=example +after_success: + - coveralls diff --git a/setup.py b/setup.py index 58fa07d..c2527a2 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ def readme(): keywords="skeleton", packages=["example"], install_requires=["docopt"], - extras_require={"test": ["pre-commit", "pytest", "pytest-cov"]}, + extras_require={"test": ["pre-commit", "pytest", "pytest-cov", "coveralls"]}, # Conveniently allows one to run the CLI tool as `example` entry_points={"console_scripts": ["example = example.example:main"]}, ) From 828e4eefcd80385d0029ad74a0e776ee1a9e7792 Mon Sep 17 00:00:00 2001 From: Felddy Date: Sun, 5 May 2019 00:45:10 -0400 Subject: [PATCH 009/282] Add coverage shield --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c2914c5..588ac1b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # skeleton-python-library # [![Build Status](https://travis-ci.com/cisagov/skeleton-python-library.svg?branch=develop)](https://travis-ci.com/cisagov/skeleton-python-library) +[![Coverage Status](https://coveralls.io/repos/github/cisagov/skeleton-python-library/badge.svg?branch=develop)](https://coveralls.io/github/cisagov/skeleton-python-library?branch=develop) [![Total alerts](https://img.shields.io/lgtm/alerts/g/cisagov/skeleton-python-library.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/skeleton-python-library/alerts/) [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/cisagov/skeleton-python-library.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/skeleton-python-library/context:python) From 8a27b48fce69a19f215f30a2d8bbd8dfe4f08f46 Mon Sep 17 00:00:00 2001 From: Felddy Date: Sun, 5 May 2019 01:13:58 -0400 Subject: [PATCH 010/282] Add missing term to coverage report --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0fa9cfa..81827e0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,6 @@ install: - pip install --upgrade -r requirements-test.txt script: - pre-commit run --all-files - - pytest -v --cov=example + - pytest -v --cov=example --cov-report term-missing after_success: - coveralls From 8050005782c7f69699c8c7084c249315a15a55f1 Mon Sep 17 00:00:00 2001 From: Felddy Date: Sun, 5 May 2019 10:50:13 -0400 Subject: [PATCH 011/282] Add configurations for pytest and pytest-cov --- .coveragerc | 12 ++++++++++++ .travis.yml | 2 +- pytest.ini | 2 ++ 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 .coveragerc create mode 100644 pytest.ini diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..982fc52 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,12 @@ +# This is the configuration for code coverage checks +# https://coverage.readthedocs.io/en/latest/config.html + +[run] +source = example +omit = +branch = true + +[report] +exclude_lines = + if __name__ == "__main__": +show_missing = true diff --git a/.travis.yml b/.travis.yml index 81827e0..a5db1c0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,6 @@ install: - pip install --upgrade -r requirements-test.txt script: - pre-commit run --all-files - - pytest -v --cov=example --cov-report term-missing + - pytest after_success: - coveralls diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..e6ce703 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = -v --cov From 9dfed077a09b007b89cec348134b04f19f2ae0d9 Mon Sep 17 00:00:00 2001 From: Felddy Date: Sun, 5 May 2019 10:51:26 -0400 Subject: [PATCH 012/282] Standardize version information source --- example/example.py | 5 +++-- setup.py | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/example/example.py b/example/example.py index 6382d8c..36f130f 100755 --- a/example/example.py +++ b/example/example.py @@ -18,6 +18,8 @@ import docopt +import example # to access __version__ + def example_div(x, y): """Print some logging messages.""" @@ -31,8 +33,7 @@ def example_div(x, y): def main(): """Set up logging and call the example function.""" - args = docopt.docopt(__doc__, version="0.0.1") - + args = docopt.docopt(__doc__, version=example.__version__) # Set up logging log_level = args["--log-level"] try: diff --git a/setup.py b/setup.py index c2527a2..26f37e2 100644 --- a/setup.py +++ b/setup.py @@ -9,6 +9,8 @@ from setuptools import setup +from example import __version__ + def readme(): """Read in and return the contents of the project's README.md file.""" @@ -19,7 +21,7 @@ def readme(): setup( name="example", # Versions should comply with PEP440 - version="0.0.1", + version=__version__, description="Example python library", long_description=readme(), long_description_content_type="text/markdown", From 92576a723025465b363e8495ec45fa3136f5617a Mon Sep 17 00:00:00 2001 From: Felddy Date: Sun, 5 May 2019 10:55:28 -0400 Subject: [PATCH 013/282] Add a success return code --- example/example.py | 1 + 1 file changed, 1 insertion(+) diff --git a/example/example.py b/example/example.py index 36f130f..9280104 100755 --- a/example/example.py +++ b/example/example.py @@ -51,6 +51,7 @@ def main(): # Stop logging and clean up logging.shutdown() + return 0 if __name__ == "__main__": From 42899a3208b825b62d2188f3ac78c8dd3080b68e Mon Sep 17 00:00:00 2001 From: Felddy Date: Sun, 5 May 2019 11:00:53 -0400 Subject: [PATCH 014/282] Add more tests and disable bandit assert use check --- .bandit.yml | 2 +- tests/test_example.py | 45 ++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/.bandit.yml b/.bandit.yml index 46bab94..5fd57d7 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -9,4 +9,4 @@ tests: #- B102 skips: - #- B101 # skip "assert used" check since assertions are required in pytests + - B101 # skip "assert used" check since assertions are required in pytests diff --git a/tests/test_example.py b/tests/test_example.py index fe3bfb8..e163c69 100644 --- a/tests/test_example.py +++ b/tests/test_example.py @@ -1,9 +1,13 @@ #!/usr/bin/env pytest -vs """Tests for example.""" +import logging +import sys +from unittest.mock import patch + import pytest -from example import example_div +import example div_params = [ (1, 1, 1), @@ -13,15 +17,50 @@ pytest.param(0, 0, 0, marks=pytest.mark.xfail(raises=ZeroDivisionError)), ] +log_levels = ( + "debug", + "info", + "warning", + "error", + "critical", + pytest.param("high2", marks=pytest.mark.xfail), +) + + +def test_version(capsys): + """Verify that version is output, and agrees with the module.""" + with pytest.raises(SystemExit): + with patch.object(sys, "argv", ["bogus", "--version"]): + example.example.main() + captured = capsys.readouterr() + assert ( + captured.out == f"{example.__version__}\n" + ), "standard output by '--version' should agree with module.__version__" + + +@pytest.mark.parametrize("level", log_levels) +def test_log_levels(level): + """Validate commandline log-level arguments.""" + with patch.object(sys, "argv", ["bogus", f"--log-level={level}"]): + with patch.object(logging.root, "handlers", []): + assert ( + logging.root.hasHandlers() is False + ), "root logger should not have handlers yet" + return_code = example.example.main() + assert ( + logging.root.hasHandlers() is True + ), "root logger should now have a handler" + assert return_code == 0, "main() should return success (0)" + @pytest.mark.parametrize("dividend, divisor, quotient", div_params) def test_division(dividend, divisor, quotient): """Verify division results.""" - result = example_div(dividend, divisor) + result = example.example_div(dividend, divisor) assert result == quotient, "result should equal quotient" # nosec def test_zero_division(): """Verify that division by zero throws the correct exception.""" with pytest.raises(ZeroDivisionError): - example_div(1, 0) + example.example_div(1, 0) From 5965f0e0715eb6e66c4a35c0764ff39b12c83d3d Mon Sep 17 00:00:00 2001 From: Felddy Date: Sun, 5 May 2019 11:02:30 -0400 Subject: [PATCH 015/282] Rename foolish log level --- tests/test_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_example.py b/tests/test_example.py index e163c69..8422af7 100644 --- a/tests/test_example.py +++ b/tests/test_example.py @@ -23,7 +23,7 @@ "warning", "error", "critical", - pytest.param("high2", marks=pytest.mark.xfail), + pytest.param("critical2", marks=pytest.mark.xfail), ) From 7c4e5658a18846f81421d7a1fb5c13506b37564e Mon Sep 17 00:00:00 2001 From: Felddy Date: Sun, 5 May 2019 11:07:26 -0400 Subject: [PATCH 016/282] Back out changes to setup.py version fetch... chicken and egg. --- setup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 26f37e2..c2527a2 100644 --- a/setup.py +++ b/setup.py @@ -9,8 +9,6 @@ from setuptools import setup -from example import __version__ - def readme(): """Read in and return the contents of the project's README.md file.""" @@ -21,7 +19,7 @@ def readme(): setup( name="example", # Versions should comply with PEP440 - version=__version__, + version="0.0.1", description="Example python library", long_description=readme(), long_description_content_type="text/markdown", From 62bbd80e7f4e36a7509cd4a40f77b8a724b16472 Mon Sep 17 00:00:00 2001 From: Felddy Date: Sun, 5 May 2019 11:44:37 -0400 Subject: [PATCH 017/282] Add ignore for pytest cache directory --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 3f73e7f..724760e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ __pycache__ .python-version .coverage +.pytest_cache From 797a699e42c6e86a7a0af71f5fb23a2b7890d0bd Mon Sep 17 00:00:00 2001 From: Felddy Date: Sun, 5 May 2019 11:46:13 -0400 Subject: [PATCH 018/282] Convert to a src layout. See: https://blog.ionelmc.ro/2014/05/25/python-packaging/#the-structure --- setup.py | 11 +++++++++-- {example => src/example}/__init__.py | 0 {example => src/example}/example.py | 0 3 files changed, 9 insertions(+), 2 deletions(-) rename {example => src/example}/__init__.py (100%) rename {example => src/example}/example.py (100%) diff --git a/setup.py b/setup.py index c2527a2..51a432a 100644 --- a/setup.py +++ b/setup.py @@ -5,9 +5,13 @@ - https://packaging.python.org/distributing/ - https://github.com/pypa/sampleproject/blob/master/setup.py +- https://blog.ionelmc.ro/2014/05/25/python-packaging/#the-structure """ -from setuptools import setup +from glob import glob +from os.path import splitext, basename + +from setuptools import setup, find_packages def readme(): @@ -50,7 +54,10 @@ def readme(): ], # What does your project relate to? keywords="skeleton", - packages=["example"], + packages=find_packages(where="src"), + package_dir={"": "src"}, + py_modules=[splitext(basename(path))[0] for path in glob("src/*.py")], + include_package_data=True, install_requires=["docopt"], extras_require={"test": ["pre-commit", "pytest", "pytest-cov", "coveralls"]}, # Conveniently allows one to run the CLI tool as `example` diff --git a/example/__init__.py b/src/example/__init__.py similarity index 100% rename from example/__init__.py rename to src/example/__init__.py diff --git a/example/example.py b/src/example/example.py similarity index 100% rename from example/example.py rename to src/example/example.py From 96591da4b763ff04171fe2112568616f5f21e583 Mon Sep 17 00:00:00 2001 From: Felddy Date: Sun, 5 May 2019 14:21:54 -0400 Subject: [PATCH 019/282] Add option to pytest to show extra test summary info --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index e6ce703..ed958e0 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,2 @@ [pytest] -addopts = -v --cov +addopts = -v -ra --cov From 7394753dff8ac5e4c5c033f3b679358577eb4d5a Mon Sep 17 00:00:00 2001 From: Felddy Date: Sun, 5 May 2019 14:24:16 -0400 Subject: [PATCH 020/282] Add sample conftest file. --- tests/conftest.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 tests/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..b060645 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,23 @@ +"""pytest plugin configuration. + +https://docs.pytest.org/en/latest/writing_plugins.html#conftest-py-plugins +""" +import pytest + + +def pytest_addoption(parser): + """Add new commandline options to pytest.""" + parser.addoption( + "--runslow", action="store_true", default=False, help="run slow tests" + ) + + +def pytest_collection_modifyitems(config, items): + """Modify collected tests based on custom marks and commandline options.""" + if config.getoption("--runslow"): + # --runslow given in cli: do not skip slow tests + return + skip_slow = pytest.mark.skip(reason="need --runslow option to run") + for item in items: + if "slow" in item.keywords: + item.add_marker(skip_slow) From da671bbfffd02da400f85333cc9f08d71def82d7 Mon Sep 17 00:00:00 2001 From: Felddy Date: Sun, 5 May 2019 14:24:36 -0400 Subject: [PATCH 021/282] Clarify comment. --- tests/test_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_example.py b/tests/test_example.py index 8422af7..17f7fac 100644 --- a/tests/test_example.py +++ b/tests/test_example.py @@ -28,7 +28,7 @@ def test_version(capsys): - """Verify that version is output, and agrees with the module.""" + """Verify that version string sent to stdout, and agrees with the module.""" with pytest.raises(SystemExit): with patch.object(sys, "argv", ["bogus", "--version"]): example.example.main() From 8e4e79d6e18c4be07cbf3968c52c1d0ec40ec9db Mon Sep 17 00:00:00 2001 From: Felddy Date: Sun, 5 May 2019 14:25:00 -0400 Subject: [PATCH 022/282] Make pytest source declaration more explicit and less magical --- .coveragerc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index 982fc52..d315b87 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,7 +2,7 @@ # https://coverage.readthedocs.io/en/latest/config.html [run] -source = example +source = src/example omit = branch = true From a748e45d3dddda0a40bd59a20d7e07ea9266dfb2 Mon Sep 17 00:00:00 2001 From: Felddy Date: Sun, 5 May 2019 14:30:28 -0400 Subject: [PATCH 023/282] Remove bandit nosec comment since we now configure bandit --- tests/test_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_example.py b/tests/test_example.py index 17f7fac..bf4448e 100644 --- a/tests/test_example.py +++ b/tests/test_example.py @@ -57,7 +57,7 @@ def test_log_levels(level): def test_division(dividend, divisor, quotient): """Verify division results.""" result = example.example_div(dividend, divisor) - assert result == quotient, "result should equal quotient" # nosec + assert result == quotient, "result should equal quotient" def test_zero_division(): From 96076b481dfd0e1936077e30cd0ae5e437119ab0 Mon Sep 17 00:00:00 2001 From: Felddy Date: Sun, 5 May 2019 14:30:51 -0400 Subject: [PATCH 024/282] Add a slow test for demonstration. --- tests/test_example.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_example.py b/tests/test_example.py index bf4448e..132dce0 100644 --- a/tests/test_example.py +++ b/tests/test_example.py @@ -60,6 +60,20 @@ def test_division(dividend, divisor, quotient): assert result == quotient, "result should equal quotient" +@pytest.mark.slow +def test_slow_division(): + """Example of using a custom marker. + + This test will only be run if --runslow is passed to pytest. + Look in conftest.py to see how this is implemented. + """ + import time + + result = example.example_div(256, 16) + time.sleep(4) + assert result == 16, "result should equal be 16" + + def test_zero_division(): """Verify that division by zero throws the correct exception.""" with pytest.raises(ZeroDivisionError): From eb2a0a592b73c0770ce1875bee451f43bc5ec81e Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 6 May 2019 10:00:48 -0400 Subject: [PATCH 025/282] Add section about pytest to contribution document. --- CONTRIBUTING.md | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b48c086..60d0b87 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,10 +27,10 @@ one. If you choose to [submit a pull request](https://github.com/cisagov/skeleton-python-library/pulls), you will notice that our continuous integration (CI) system runs a -fairly extensive set of linters and syntax checkers. Your pull -request may fail these checks, and that's OK. If you want you can -stop there and wait for us to make the necessary corrections to ensure -your code passes the CI checks. +fairly extensive set of linters, syntax checkers, system, and unit tests. +Your pull request may fail these checks, and that's OK. If you want +you can stop there and wait for us to make the necessary corrections +to ensure your code passes the CI checks. If you want to make the changes yourself, or if you want to become a regular contributor, then you will want to set up @@ -96,6 +96,17 @@ At this point the pre-commit checks will run against any files that you attempt to commit. If you want to run the checks against the entire repo, just execute `pre-commit run --all-files`. +### Running unit and system tests ### + +In addition to the pre-commit checks the CI system will run the suite +of unit and system tests that are included with this project. To run +these tests locally execute `pytest` from the root of the project. + +We encourage any updates to these tests to improve the overall code +coverage. If your pull request adds new functionality we would +appreciate it if you extend existing test cases, or add new ones to +exercise the newly added code. + ## Public domain ## This project is in the public domain within the United States, and From 19afa70f67a10f14d76b9132f508849146b35656 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 6 May 2019 10:44:34 -0400 Subject: [PATCH 026/282] Run pre-commit autoupdate Also update the URL for black since it moved GitHub orgs from ambv to python. --- .pre-commit-config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2a03327..0bac8c8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ --- repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.1.0 + rev: v2.2.1 hooks: - id: check-executables-have-shebangs - id: check-json @@ -23,7 +23,7 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.14.1 + rev: v0.15.0 hooks: - id: markdownlint # The LICENSE.md must match the license text exactly for @@ -45,7 +45,7 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.14.0 + rev: v1.16.3 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit @@ -54,7 +54,7 @@ repos: - id: bandit args: - --config=.bandit.yml - - repo: https://github.com/ambv/black + - repo: https://github.com/python/black rev: 19.3b0 hooks: - id: black @@ -74,6 +74,6 @@ repos: hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier - rev: 1.16.2 + rev: 1.17.0 hooks: - id: prettier From eb4fbd751b57db9c728e7b3065edbfde50633d97 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 6 May 2019 11:16:05 -0400 Subject: [PATCH 027/282] Make .bandit.yml pass yamllint --- .bandit.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.bandit.yml b/.bandit.yml index 46bab94..8ba42d1 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -1,3 +1,4 @@ +--- # Configuration file for the Bandit python security scanner # https://bandit.readthedocs.io/en/latest/config.html @@ -5,8 +6,8 @@ # If `tests` is empty, all tests are are considered included. tests: - #- B101 - #- B102 +# - B101 +# - B102 skips: - #- B101 # skip "assert used" check since assertions are required in pytests +# - B101 # skip "assert used" check since assertions are required in pytests From 4f8b3fa5cd8954aeb58ca8c951910e7362043b9e Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 6 May 2019 12:31:40 -0400 Subject: [PATCH 028/282] Improve version handling --- setup.py | 10 +++++++++- src/example/__init__.py | 2 +- src/example/_version.py | 2 ++ src/example/example.py | 4 ++-- 4 files changed, 14 insertions(+), 4 deletions(-) create mode 100644 src/example/_version.py diff --git a/setup.py b/setup.py index 51a432a..fd03f27 100644 --- a/setup.py +++ b/setup.py @@ -20,10 +20,18 @@ def readme(): return f.read() +def package_vars(version_file): + """Read in and return the variables defined by the version_file.""" + pkg_vars = {} + with open(version_file) as f: + exec(f.read(), pkg_vars) # nosec + return pkg_vars + + setup( name="example", # Versions should comply with PEP440 - version="0.0.1", + version=package_vars("src/example/_version.py")["__version__"], description="Example python library", long_description=readme(), long_description_content_type="text/markdown", diff --git a/src/example/__init__.py b/src/example/__init__.py index 0630045..ba27a7c 100644 --- a/src/example/__init__.py +++ b/src/example/__init__.py @@ -1,5 +1,5 @@ """The example library.""" from .example import example_div +from ._version import __version__ # noqa: F401 __all__ = ["example_div"] -__version__ = "0.0.1" diff --git a/src/example/_version.py b/src/example/_version.py new file mode 100644 index 0000000..33cee84 --- /dev/null +++ b/src/example/_version.py @@ -0,0 +1,2 @@ +"""This file defines the version of this module.""" +__version__ = "0.0.1" diff --git a/src/example/example.py b/src/example/example.py index 9280104..75335e7 100755 --- a/src/example/example.py +++ b/src/example/example.py @@ -18,7 +18,7 @@ import docopt -import example # to access __version__ +from ._version import __version__ def example_div(x, y): @@ -33,7 +33,7 @@ def example_div(x, y): def main(): """Set up logging and call the example function.""" - args = docopt.docopt(__doc__, version=example.__version__) + args = docopt.docopt(__doc__, version=__version__) # Set up logging log_level = args["--log-level"] try: From c934af332742e6324c49729b03c9e50de2b0360b Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 6 May 2019 14:36:59 -0400 Subject: [PATCH 029/282] Split bandit pre-commit checks into two diffrent runs One for tests and one for everthing else. --- .bandit.yml | 1 + .pre-commit-config.yaml | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/.bandit.yml b/.bandit.yml index 5ae161d..7b89269 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -1,6 +1,7 @@ --- # Configuration file for the Bandit python security scanner # https://bandit.readthedocs.io/en/latest/config.html +# This config is applied to bandit when scanning the "tests" tree # Tests are first included by `tests`, and then excluded by `skips`. # If `tests` is empty, all tests are are considered included. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0bac8c8..a672df8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,12 +48,22 @@ repos: rev: v1.16.3 hooks: - id: pyupgrade + # Run bandit on "tests" tree with a configuration - repo: https://github.com/PyCQA/bandit rev: 2a1dbab hooks: - id: bandit + name: bandit (tests tree) + files: tests args: - --config=.bandit.yml + # Run bandit everything but tests directory + - repo: https://github.com/PyCQA/bandit + rev: 2a1dbab + hooks: + - id: bandit + name: bandit (everything else) + exclude: tests - repo: https://github.com/python/black rev: 19.3b0 hooks: From a9ff3a336c0e9c9ee153b22a0de3bf5d07f5e462 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Sun, 26 May 2019 11:49:13 -0400 Subject: [PATCH 030/282] Enable TravisCI caching for pip and pre-commit plugins This should speed up TravisCI builds. --- .travis.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.travis.yml b/.travis.yml index a4493e4..2c70e21 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,8 +2,15 @@ dist: xenial language: python python: 3.7 +# pre-commit hooks can use Docker, so we should go ahead and enable it services: docker +# Cache pip packages and pre-commit plugins to speed up builds +cache: + pip: true + directories: + - $HOME/.cache/pre-commit + install: - pip install --upgrade -r requirements-test.txt script: From 4e9280eb2baf8a72fa7abdf03898efb8ca82e2a1 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Sun, 26 May 2019 11:57:27 -0400 Subject: [PATCH 031/282] Update pre-commit plugins to latest versions Via pre-commit autoupdate. --- .pre-commit-config.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0bac8c8..b6a0939 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ --- repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.2.1 + rev: v2.2.3 hooks: - id: check-executables-have-shebangs - id: check-json @@ -23,7 +23,7 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.15.0 + rev: v0.16.0 hooks: - id: markdownlint # The LICENSE.md must match the license text exactly for @@ -35,7 +35,7 @@ repos: hooks: - id: yamllint - repo: https://github.com/detailyang/pre-commit-shell - rev: 1.0.4 + rev: 1.0.5 hooks: - id: shell-lint - repo: https://gitlab.com/pycqa/flake8 @@ -45,11 +45,11 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.16.3 + rev: v1.17.1 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit - rev: 2a1dbab + rev: 1.6.0 hooks: - id: bandit args: @@ -74,6 +74,6 @@ repos: hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier - rev: 1.17.0 + rev: 1.17.1 hooks: - id: prettier From 59b0be84c3ff59c57dc44ee1e311a9a9144519dc Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 6 Jun 2019 13:58:50 -0400 Subject: [PATCH 032/282] Add data to the package. This shows how to added data to a package and correctly access it. --- setup.py | 1 + src/example/data/secret.txt | 1 + 2 files changed, 2 insertions(+) create mode 100644 src/example/data/secret.txt diff --git a/setup.py b/setup.py index fd03f27..737435a 100644 --- a/setup.py +++ b/setup.py @@ -64,6 +64,7 @@ def package_vars(version_file): keywords="skeleton", packages=find_packages(where="src"), package_dir={"": "src"}, + package_data={"example": ["data/*.txt"]}, py_modules=[splitext(basename(path))[0] for path in glob("src/*.py")], include_package_data=True, install_requires=["docopt"], diff --git a/src/example/data/secret.txt b/src/example/data/secret.txt new file mode 100644 index 0000000..c40a49b --- /dev/null +++ b/src/example/data/secret.txt @@ -0,0 +1 @@ +Three may keep a secret, if two of them are dead. From 51642cb2b993908f41abb647f76de6570c935baf Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 6 Jun 2019 13:59:21 -0400 Subject: [PATCH 033/282] Add reading of package data and environment --- src/example/example.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/example/example.py b/src/example/example.py index 75335e7..d389e0d 100755 --- a/src/example/example.py +++ b/src/example/example.py @@ -14,12 +14,16 @@ """ import logging +import os import sys import docopt +import pkg_resources from ._version import __version__ +DEFAULT_ECHO_MESSAGE = "Hello World from the example default!" + def example_div(x, y): """Print some logging messages.""" @@ -47,7 +51,19 @@ def main(): ) return 1 - print(example_div(8, 2)) + print(f"8 / 2 == {example_div(8, 2)}") + + # Access some data from an environment variable + message = os.getenv("ECHO_MESSAGE", DEFAULT_ECHO_MESSAGE) + print(f'ECHO_MESSAGE="{message}"') + + # Access some data from our package data (see the setup.py) + secret_message = ( + pkg_resources.resource_string("example", "data/secret.txt") + .decode("utf-8") + .strip() + ) + print(f'Secret="{secret_message}"') # Stop logging and clean up logging.shutdown() From b58fa62c004d4ffe42188f258653211ee85d6405 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Thu, 6 Jun 2019 16:12:58 -0400 Subject: [PATCH 034/282] Add changes from running pre-commit autoupdate --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b6a0939..e9029c0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -45,7 +45,7 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.17.1 + rev: v1.18.0 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit @@ -64,7 +64,7 @@ repos: - id: ansible-lint # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git - rev: v1.11.0 + rev: v1.12.0 hooks: - id: terraform_fmt - id: terraform_validate_no_variables From 91ef4ddc241c5a8d15e2307d3c12aa6c11098b71 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 6 Jun 2019 16:24:03 -0400 Subject: [PATCH 035/282] Declare setuptools explicity as a requirement for pkg_resources use. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 737435a..7570195 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,7 @@ def package_vars(version_file): package_data={"example": ["data/*.txt"]}, py_modules=[splitext(basename(path))[0] for path in glob("src/*.py")], include_package_data=True, - install_requires=["docopt"], + install_requires=["docopt", "setuptools"], extras_require={"test": ["pre-commit", "pytest", "pytest-cov", "coveralls"]}, # Conveniently allows one to run the CLI tool as `example` entry_points={"console_scripts": ["example = example.example:main"]}, From 740457694ab4f14fbe89583f14de86aab064765b Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Thu, 6 Jun 2019 16:33:07 -0400 Subject: [PATCH 036/282] Remove the terraform-docs hook terraform-docs is currently broken with respect to terraform 0.12. --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e9029c0..dd98ddc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -68,7 +68,6 @@ repos: hooks: - id: terraform_fmt - id: terraform_validate_no_variables - - id: terraform_docs - repo: https://github.com/IamTheFij/docker-pre-commit rev: v1.0.0 hooks: From bd4206bbcd94f00aaa3ae8a4b1daf8db002120f1 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 7 Jun 2019 11:19:41 -0400 Subject: [PATCH 037/282] Configure prettier to ignore JSON files JSON files are already being linted by the pretty-format-json hook. --- .prettierignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.prettierignore b/.prettierignore index f1e840c..738d402 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,3 +1,5 @@ +# Already being linted by pretty-format-json +*.json # Already being linted by mdl *.md # Already being linted by yamllint From 95d2d62cd8ae1c14322ae8128a240de00092a99e Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 19 Jun 2019 09:10:50 -0400 Subject: [PATCH 038/282] Add Snyk badge/shield --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 588ac1b..430bd0b 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ [![Coverage Status](https://coveralls.io/repos/github/cisagov/skeleton-python-library/badge.svg?branch=develop)](https://coveralls.io/github/cisagov/skeleton-python-library?branch=develop) [![Total alerts](https://img.shields.io/lgtm/alerts/g/cisagov/skeleton-python-library.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/skeleton-python-library/alerts/) [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/cisagov/skeleton-python-library.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/skeleton-python-library/context:python) +[![Known Vulnerabilities](https://snyk.io/test/github/cisagov/skeleton-python-library/develop/badge.svg)](https://snyk.io/test/github/cisagov/skeleton-python-library) This is a generic skeleton project that can be used to quickly get a new [cisagov](https://github.com/cisagov) Python library GitHub From 5bcf220cc46b2077ddcf9fbed1f2536de5735cd2 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 20 Jun 2019 11:23:20 -0400 Subject: [PATCH 039/282] Add test to verify TRAVIS_TAG agrees with project version during release --- tests/test_example.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_example.py b/tests/test_example.py index 132dce0..4415bf9 100644 --- a/tests/test_example.py +++ b/tests/test_example.py @@ -2,6 +2,7 @@ """Tests for example.""" import logging +import os import sys from unittest.mock import patch @@ -29,13 +30,19 @@ def test_version(capsys): """Verify that version string sent to stdout, and agrees with the module.""" + project_version = example.__version__ with pytest.raises(SystemExit): with patch.object(sys, "argv", ["bogus", "--version"]): example.example.main() captured = capsys.readouterr() assert ( - captured.out == f"{example.__version__}\n" + captured.out == f"{project_version}\n" ), "standard output by '--version' should agree with module.__version__" + travis_tag = os.getenv("TRAVIS_TAG") + if travis_tag: + assert ( + travis_tag == project_version or travis_tag == f"v{project_version}" + ), "TRAVIS_TAG does not match the project version" @pytest.mark.parametrize("level", log_levels) From b42ae13683339ee060575e8ebe7e80190a022463 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 20 Jun 2019 11:53:56 -0400 Subject: [PATCH 040/282] Separate release version test from stdout version test --- tests/test_example.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/tests/test_example.py b/tests/test_example.py index 4415bf9..64ed08e 100644 --- a/tests/test_example.py +++ b/tests/test_example.py @@ -27,22 +27,30 @@ pytest.param("critical2", marks=pytest.mark.xfail), ) +# define sources of version strings +TRAVIS_TAG = os.getenv("TRAVIS_TAG") +PROJECT_VERSION = example.__version__ -def test_version(capsys): - """Verify that version string sent to stdout, and agrees with the module.""" - project_version = example.__version__ + +def test_stdout_version(capsys): + """Verify that version string sent to stdout agrees with the module version.""" with pytest.raises(SystemExit): with patch.object(sys, "argv", ["bogus", "--version"]): example.example.main() captured = capsys.readouterr() assert ( - captured.out == f"{project_version}\n" + captured.out == f"{PROJECT_VERSION}\n" ), "standard output by '--version' should agree with module.__version__" - travis_tag = os.getenv("TRAVIS_TAG") - if travis_tag: - assert ( - travis_tag == project_version or travis_tag == f"v{project_version}" - ), "TRAVIS_TAG does not match the project version" + + +@pytest.mark.skipif( + TRAVIS_TAG in [None, ""], reason="this is not a release (TRAVIS_TAG not set)" +) +def test_release_version(): + """Verify that release tag version agrees with the module version.""" + assert ( + TRAVIS_TAG == PROJECT_VERSION or TRAVIS_TAG == f"v{PROJECT_VERSION}" + ), "TRAVIS_TAG does not match the project version" @pytest.mark.parametrize("level", log_levels) From 75c4f524ddc6c44d106ca4f2d0466f39b37cf3c0 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 20 Jun 2019 12:15:25 -0400 Subject: [PATCH 041/282] TRAVIS_TAG must start with a "v" --- tests/test_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_example.py b/tests/test_example.py index 64ed08e..9b9ec4f 100644 --- a/tests/test_example.py +++ b/tests/test_example.py @@ -49,7 +49,7 @@ def test_stdout_version(capsys): def test_release_version(): """Verify that release tag version agrees with the module version.""" assert ( - TRAVIS_TAG == PROJECT_VERSION or TRAVIS_TAG == f"v{PROJECT_VERSION}" + TRAVIS_TAG == f"v{PROJECT_VERSION}" ), "TRAVIS_TAG does not match the project version" From c6f2adb6abc41b0bbb6abae88aba10cb6ab2e515 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell Date: Thu, 20 Jun 2019 15:14:12 -0400 Subject: [PATCH 042/282] Update two text blocks to use the console tag instead of the bash tag for formatting. --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ede2f47..117baba 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -77,7 +77,7 @@ Once `pyenv` and `pyenv-virtualenv` are installed on your system, you can create and configure the Python virtual environment with these commands: -```bash +```console cd skeleton-generic pyenv virtualenv skeleton-generic pyenv local skeleton-generic @@ -88,7 +88,7 @@ pip install -r requirements-dev.txt Now setting up pre-commit is as simple as: -```bash +```console pre-commit install ``` From a13597d2330f947797fce78bed3ac665c0623ddc Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 24 Jun 2019 17:11:45 -0400 Subject: [PATCH 043/282] Update pre-commit hooks via pre-commit autoupdate --- .pre-commit-config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dd98ddc..eeddaed 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,7 +23,7 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.16.0 + rev: v0.17.0 hooks: - id: markdownlint # The LICENSE.md must match the license text exactly for @@ -31,7 +31,7 @@ repos: # alone. exclude: LICENSE.md - repo: https://github.com/adrienverge/yamllint - rev: v1.15.0 + rev: v1.16.0 hooks: - id: yamllint - repo: https://github.com/detailyang/pre-commit-shell @@ -45,11 +45,11 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.18.0 + rev: v1.19.0 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit - rev: 1.6.0 + rev: 1.6.1 hooks: - id: bandit args: @@ -73,6 +73,6 @@ repos: hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier - rev: 1.17.1 + rev: 1.18.2 hooks: - id: prettier From 1843ca0b482624b867828c95c99faa9cb01253b7 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 24 Jun 2019 17:14:02 -0400 Subject: [PATCH 044/282] Add config file for markdownlint This config file turns off the "line too long" rule when within a code block or a table. Often there is no way to avoid going over 80 lines in such cases. --- .mdl_config.json | 7 +++++++ .pre-commit-config.yaml | 2 ++ 2 files changed, 9 insertions(+) create mode 100644 .mdl_config.json diff --git a/.mdl_config.json b/.mdl_config.json new file mode 100644 index 0000000..492955a --- /dev/null +++ b/.mdl_config.json @@ -0,0 +1,7 @@ +{ + "MD013": { + "code_blocks": false, + "tables": false + }, + "default": true +} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index eeddaed..491b94c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,6 +30,8 @@ repos: # GitHub's autorecognition fu to work, so we should leave it # alone. exclude: LICENSE.md + args: + - --config=.mdl_config.json - repo: https://github.com/adrienverge/yamllint rev: v1.16.0 hooks: From 3fee0b86874070e9c9cb41a6d82b7904a3352590 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 24 Jun 2019 17:17:54 -0400 Subject: [PATCH 045/282] Rename LICENSE.md to LICENSE, make the other changes that requires The license text is not actually in Markdown, and adding the .md extension actually makes it render a little differently than intended. --- .pre-commit-config.yaml | 4 ---- CONTRIBUTING.md | 4 ++-- LICENSE.md => LICENSE | 0 README.md | 4 ++-- 4 files changed, 4 insertions(+), 8 deletions(-) rename LICENSE.md => LICENSE (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 491b94c..a4b6299 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,10 +26,6 @@ repos: rev: v0.17.0 hooks: - id: markdownlint - # The LICENSE.md must match the license text exactly for - # GitHub's autorecognition fu to work, so we should leave it - # alone. - exclude: LICENSE.md args: - --config=.mdl_config.json - repo: https://github.com/adrienverge/yamllint diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 117baba..93addc2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,8 +8,8 @@ of contribution, and don't want a wall of rules to get in the way of that. Before contributing, we encourage you to read our CONTRIBUTING policy -(you are here), our [LICENSE](LICENSE.md), and our -[README](README.md), all of which should be in this repository. +(you are here), our [LICENSE](LICENSE), and our [README](README.md), +all of which should be in this repository. ## Issues ## diff --git a/LICENSE.md b/LICENSE similarity index 100% rename from LICENSE.md rename to LICENSE diff --git a/README.md b/README.md index 2337970..b880f4d 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This is a generic skeleton project that can be used to quickly get a new [cisagov](https://github.com/cisagov) GitHub project started. -This skeleton project contains [licensing information](LICENSE.md), as +This skeleton project contains [licensing information](LICENSE), as well as [pre-commit hooks](https://pre-commit.com) and a [Travis CI](https://travis-ci.com) configuration appropriate for the major languages that we use. @@ -19,7 +19,7 @@ details. ## License ## -This project is in the worldwide [public domain](LICENSE.md). +This project is in the worldwide [public domain](LICENSE). This project is in the public domain within the United States, and copyright and related rights in the work worldwide are waived through From db11980db15135db28fa4c023417c4b1633f1376 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 5 Jul 2019 17:47:01 -0400 Subject: [PATCH 046/282] Register mark in heed of pytest warning. https://docs.pytest.org/en/latest/mark.html --- tests/conftest.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index b060645..58c399f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,6 +12,11 @@ def pytest_addoption(parser): ) +def pytest_configure(config): + """Register new markers.""" + config.addinivalue_line("markers", "slow: mark test as slow") + + def pytest_collection_modifyitems(config, items): """Modify collected tests based on custom marks and commandline options.""" if config.getoption("--runslow"): From a675360e21c7d712f4c98dc732d87289f5589c07 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 19 Jul 2019 16:56:45 -0400 Subject: [PATCH 047/282] Open the README file with utf-8 encoding. Backported from pca-gophish-composition. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7570195..eb246c4 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ def readme(): """Read in and return the contents of the project's README.md file.""" - with open("README.md") as f: + with open("README.md", encoding="utf-8") as f: return f.read() From 2932d35483cb8b7be71da69b7006e58298846792 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 11 Sep 2019 15:58:44 -0400 Subject: [PATCH 048/282] Add updates from running pre-commit autoupdate --- .pre-commit-config.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a4b6299..9580afa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ --- repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.2.3 + rev: v2.3.0 hooks: - id: check-executables-have-shebangs - id: check-json @@ -23,13 +23,13 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.17.0 + rev: v0.18.0 hooks: - id: markdownlint args: - --config=.mdl_config.json - repo: https://github.com/adrienverge/yamllint - rev: v1.16.0 + rev: v1.17.0 hooks: - id: yamllint - repo: https://github.com/detailyang/pre-commit-shell @@ -37,17 +37,17 @@ repos: hooks: - id: shell-lint - repo: https://gitlab.com/pycqa/flake8 - rev: 3.7.7 + rev: 3.7.8 hooks: - id: flake8 additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.19.0 + rev: v1.23.0 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit - rev: 1.6.1 + rev: 1.6.2 hooks: - id: bandit args: @@ -57,7 +57,7 @@ repos: hooks: - id: black - repo: https://github.com/ansible/ansible-lint.git - rev: v4.1.0a0 + rev: v4.1.1a0 hooks: - id: ansible-lint # files: molecule/default/playbook.yml From a68f7e8e913cee322c613f903cf4a5b9902b81bc Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 17 Oct 2019 13:53:16 -0400 Subject: [PATCH 049/282] Add a actions workflow similar to the travis config. --- .github/workflows/pythonapp.yml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/workflows/pythonapp.yml diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml new file mode 100644 index 0000000..f2f9486 --- /dev/null +++ b/.github/workflows/pythonapp.yml @@ -0,0 +1,31 @@ +--- +name: Python application + +on: [push] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade -r requirements-test.txt + + - name: Run pre-commit on all files + run: pre-commit run --all-files + + - name: Test with pytest + run: pytest + + - name: Report test coverage + run: coveralls + if: success() From 585d43b80cc461c3cca824b3992ab89cc6757506 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 17 Oct 2019 14:30:39 -0400 Subject: [PATCH 050/282] Change python version to 3.8 --- .github/workflows/pythonapp.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index f2f9486..4281aa8 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -10,10 +10,10 @@ jobs: steps: - uses: actions/checkout@v1 - - name: Set up Python 3.7 + - name: Set up Python 3.8 uses: actions/setup-python@v1 with: - python-version: 3.7 + python-version: 3.8 - name: Install dependencies run: | From bba0aa53c3555524f1efca55a582fc16c5711bd5 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 17 Oct 2019 14:33:29 -0400 Subject: [PATCH 051/282] Revert to python 3.7 --- .github/workflows/pythonapp.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 4281aa8..f2f9486 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -10,10 +10,10 @@ jobs: steps: - uses: actions/checkout@v1 - - name: Set up Python 3.8 + - name: Set up Python 3.7 uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: 3.7 - name: Install dependencies run: | From dd0e2f704a64937adc5e1ef68ad569d5d810190c Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 17 Oct 2019 14:40:18 -0400 Subject: [PATCH 052/282] Work around pre-commit issue. See: https://github.com/pypa/setuptools/issues/1869 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index eb246c4..61dc7dd 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,7 @@ def package_vars(version_file): package_data={"example": ["data/*.txt"]}, py_modules=[splitext(basename(path))[0] for path in glob("src/*.py")], include_package_data=True, - install_requires=["docopt", "setuptools"], + install_requires=["docopt", "setuptools=41.0.0"], extras_require={"test": ["pre-commit", "pytest", "pytest-cov", "coveralls"]}, # Conveniently allows one to run the CLI tool as `example` entry_points={"console_scripts": ["example = example.example:main"]}, From 47dc2802d81a6ea10c6baf170d16ec0b0353efe3 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 17 Oct 2019 14:44:49 -0400 Subject: [PATCH 053/282] Fix install_requires syntax. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 61dc7dd..ad1ffb6 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,7 @@ def package_vars(version_file): package_data={"example": ["data/*.txt"]}, py_modules=[splitext(basename(path))[0] for path in glob("src/*.py")], include_package_data=True, - install_requires=["docopt", "setuptools=41.0.0"], + install_requires=["docopt", "setuptools==41.0.0"], extras_require={"test": ["pre-commit", "pytest", "pytest-cov", "coveralls"]}, # Conveniently allows one to run the CLI tool as `example` entry_points={"console_scripts": ["example = example.example:main"]}, From 083f9daeb905ebc5c285cffa3758cf050e1457a3 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 17 Oct 2019 14:53:17 -0400 Subject: [PATCH 054/282] Update pre-commit config to lastest versions. --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6a3dd2b..e8b13fc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,7 +29,7 @@ repos: args: - --config=.mdl_config.json - repo: https://github.com/adrienverge/yamllint - rev: v1.17.0 + rev: v1.18.0 hooks: - id: yamllint - repo: https://github.com/detailyang/pre-commit-shell @@ -43,7 +43,7 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.23.0 + rev: v1.25.0 hooks: - id: pyupgrade # Run bandit on "tests" tree with a configuration @@ -57,7 +57,7 @@ repos: - --config=.bandit.yml # Run bandit everything but tests directory - repo: https://github.com/PyCQA/bandit - rev: 2a1dbab + rev: 1.6.2 hooks: - id: bandit name: bandit (everything else) From efda8d541c01ce73ef25371f3392d706eccaa553 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 17 Oct 2019 14:53:59 -0400 Subject: [PATCH 055/282] Remove setuptools pinning. This is an ansible lint issue: https://github.com/ansible/ansible-lint/issues/590 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ad1ffb6..eb246c4 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,7 @@ def package_vars(version_file): package_data={"example": ["data/*.txt"]}, py_modules=[splitext(basename(path))[0] for path in glob("src/*.py")], include_package_data=True, - install_requires=["docopt", "setuptools==41.0.0"], + install_requires=["docopt", "setuptools"], extras_require={"test": ["pre-commit", "pytest", "pytest-cov", "coveralls"]}, # Conveniently allows one to run the CLI tool as `example` entry_points={"console_scripts": ["example = example.example:main"]}, From ca380166dd47608d76fee4152a357ab9c4565535 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 17 Oct 2019 14:59:27 -0400 Subject: [PATCH 056/282] Apply workaround for ansible-lint error. See: https://github.com/ansible/ansible-lint/issues/590 --- setup.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index eb246c4..2100efe 100644 --- a/setup.py +++ b/setup.py @@ -68,7 +68,15 @@ def package_vars(version_file): py_modules=[splitext(basename(path))[0] for path in glob("src/*.py")], include_package_data=True, install_requires=["docopt", "setuptools"], - extras_require={"test": ["pre-commit", "pytest", "pytest-cov", "coveralls"]}, + extras_require={ + "test": [ + "pre-commit", + "pytest", + "pytest-cov", + "coveralls", + "virtualenv==16.3.0", + ] + }, # Conveniently allows one to run the CLI tool as `example` entry_points={"console_scripts": ["example = example.example:main"]}, ) From 1b30c8da44e11e82ea52098ab06ad7982ba3d2c0 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 17 Oct 2019 15:12:07 -0400 Subject: [PATCH 057/282] Add special sauce for coveralls. See: https://github.com/marketplace/actions/coveralls-github-action --- .github/workflows/pythonapp.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index f2f9486..c752786 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -26,6 +26,8 @@ jobs: - name: Test with pytest run: pytest - - name: Report test coverage - run: coveralls + - name: Coveralls + uses: coverallsapp/github-action@master + with: + github-token: ${{ secrets.github_token }} if: success() From 28c6ada61c3683c9eccb47d0d92d0d4c86017026 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 17 Oct 2019 15:19:11 -0400 Subject: [PATCH 058/282] Add config to tell coveralls where the coverage file lives. --- .github/workflows/pythonapp.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index c752786..d442fb6 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -28,6 +28,7 @@ jobs: - name: Coveralls uses: coverallsapp/github-action@master + path-to-lcov: .coverage with: github-token: ${{ secrets.github_token }} if: success() From 91d01649871bf0dff74a3692491a9ee336e900bd Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 17 Oct 2019 15:23:11 -0400 Subject: [PATCH 059/282] Fix syntax error. Learning is fun! --- .github/workflows/pythonapp.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index d442fb6..65638c1 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -28,7 +28,7 @@ jobs: - name: Coveralls uses: coverallsapp/github-action@master - path-to-lcov: .coverage with: github-token: ${{ secrets.github_token }} + path-to-lcov: .coverage if: success() From 9c6b5e41b8b03f65d905d1faee4c6417ae04a0c1 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 17 Oct 2019 15:53:07 -0400 Subject: [PATCH 060/282] Switch to using coveralls with token. --- .github/workflows/pythonapp.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 65638c1..97a8fe7 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -27,8 +27,7 @@ jobs: run: pytest - name: Coveralls - uses: coverallsapp/github-action@master - with: - github-token: ${{ secrets.github_token }} - path-to-lcov: .coverage + run: coveralls + env: + COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} if: success() From c6d4d5facf4bfdf2e6cc92b4f47e26a1c9df99f3 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 17 Oct 2019 16:20:09 -0400 Subject: [PATCH 061/282] Rename workflow and add badge to README. --- .github/workflows/{pythonapp.yml => build.yml} | 2 +- README.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) rename .github/workflows/{pythonapp.yml => build.yml} (96%) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/build.yml similarity index 96% rename from .github/workflows/pythonapp.yml rename to .github/workflows/build.yml index 97a8fe7..76915ad 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/build.yml @@ -1,5 +1,5 @@ --- -name: Python application +name: Build on: [push] diff --git a/README.md b/README.md index 8b7f521..18eba75 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # skeleton-python-library # -[![Build Status](https://travis-ci.com/cisagov/skeleton-python-library.svg?branch=develop)](https://travis-ci.com/cisagov/skeleton-python-library) +![GitHub Build Status](https://github.com/cisagov/skeleton-python-library/workflows/build/badge.svg) +[![Travis Build Status](https://travis-ci.com/cisagov/skeleton-python-library.svg?branch=develop)](https://travis-ci.com/cisagov/skeleton-python-library) [![Coverage Status](https://coveralls.io/repos/github/cisagov/skeleton-python-library/badge.svg?branch=develop)](https://coveralls.io/github/cisagov/skeleton-python-library?branch=develop) [![Total alerts](https://img.shields.io/lgtm/alerts/g/cisagov/skeleton-python-library.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/skeleton-python-library/alerts/) [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/cisagov/skeleton-python-library.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/skeleton-python-library/context:python) From ff27a9ad83647df0105d3e4eaad2e31b2290e980 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 17 Oct 2019 16:21:43 -0400 Subject: [PATCH 062/282] Correct capitalization for badge. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 18eba75..dc394f4 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # skeleton-python-library # -![GitHub Build Status](https://github.com/cisagov/skeleton-python-library/workflows/build/badge.svg) +![GitHub Build Status](https://github.com/cisagov/skeleton-python-library/workflows/Build/badge.svg) [![Travis Build Status](https://travis-ci.com/cisagov/skeleton-python-library.svg?branch=develop)](https://travis-ci.com/cisagov/skeleton-python-library) [![Coverage Status](https://coveralls.io/repos/github/cisagov/skeleton-python-library/badge.svg?branch=develop)](https://coveralls.io/github/cisagov/skeleton-python-library?branch=develop) [![Total alerts](https://img.shields.io/lgtm/alerts/g/cisagov/skeleton-python-library.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/skeleton-python-library/alerts/) From 58c707dedfb658373ed8060fa6f7fb09488dc7d9 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 17 Oct 2019 16:24:10 -0400 Subject: [PATCH 063/282] Make action badge a link. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dc394f4..a3704e0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # skeleton-python-library # -![GitHub Build Status](https://github.com/cisagov/skeleton-python-library/workflows/Build/badge.svg) +[![GitHub Build Status](https://github.com/cisagov/skeleton-python-library/workflows/Build/badge.svg)](https://github.com/cisagov/skeleton-python-library/actions) [![Travis Build Status](https://travis-ci.com/cisagov/skeleton-python-library.svg?branch=develop)](https://travis-ci.com/cisagov/skeleton-python-library) [![Coverage Status](https://coveralls.io/repos/github/cisagov/skeleton-python-library/badge.svg?branch=develop)](https://coveralls.io/github/cisagov/skeleton-python-library?branch=develop) [![Total alerts](https://img.shields.io/lgtm/alerts/g/cisagov/skeleton-python-library.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/skeleton-python-library/alerts/) From f8ca5c6791259e222367deee3d51a1b3cf02346f Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 17 Oct 2019 16:29:46 -0400 Subject: [PATCH 064/282] Lowercasing workflow name so the shield matches all others. --- .github/workflows/build.yml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 76915ad..b02ea85 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,5 +1,5 @@ --- -name: Build +name: build on: [push] diff --git a/README.md b/README.md index a3704e0..844c96d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # skeleton-python-library # -[![GitHub Build Status](https://github.com/cisagov/skeleton-python-library/workflows/Build/badge.svg)](https://github.com/cisagov/skeleton-python-library/actions) +[![GitHub Build Status](https://github.com/cisagov/skeleton-python-library/workflows/build/badge.svg)](https://github.com/cisagov/skeleton-python-library/actions) [![Travis Build Status](https://travis-ci.com/cisagov/skeleton-python-library.svg?branch=develop)](https://travis-ci.com/cisagov/skeleton-python-library) [![Coverage Status](https://coveralls.io/repos/github/cisagov/skeleton-python-library/badge.svg?branch=develop)](https://coveralls.io/github/cisagov/skeleton-python-library?branch=develop) [![Total alerts](https://img.shields.io/lgtm/alerts/g/cisagov/skeleton-python-library.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/skeleton-python-library/alerts/) From da7e1ccbbf2163aa395dd0cd154e5b04ee6ea37e Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 17 Oct 2019 16:39:05 -0400 Subject: [PATCH 065/282] Remove travis. :( --- .travis.yml | 20 -------------------- README.md | 9 ++++----- 2 files changed, 4 insertions(+), 25 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f4a543e..0000000 --- a/.travis.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -dist: xenial -language: python -python: 3.7 -# pre-commit hooks can use Docker, so we should go ahead and enable it -services: docker - -# Cache pip packages and pre-commit plugins to speed up builds -cache: - pip: true - directories: - - $HOME/.cache/pre-commit - -install: - - pip install --upgrade -r requirements-test.txt -script: - - pre-commit run --all-files - - pytest -after_success: - - coveralls diff --git a/README.md b/README.md index 844c96d..58440eb 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ # skeleton-python-library # [![GitHub Build Status](https://github.com/cisagov/skeleton-python-library/workflows/build/badge.svg)](https://github.com/cisagov/skeleton-python-library/actions) -[![Travis Build Status](https://travis-ci.com/cisagov/skeleton-python-library.svg?branch=develop)](https://travis-ci.com/cisagov/skeleton-python-library) [![Coverage Status](https://coveralls.io/repos/github/cisagov/skeleton-python-library/badge.svg?branch=develop)](https://coveralls.io/github/cisagov/skeleton-python-library?branch=develop) [![Total alerts](https://img.shields.io/lgtm/alerts/g/cisagov/skeleton-python-library.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/skeleton-python-library/alerts/) [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/cisagov/skeleton-python-library.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/skeleton-python-library/context:python) @@ -10,10 +9,10 @@ This is a generic skeleton project that can be used to quickly get a new [cisagov](https://github.com/cisagov) Python library GitHub project started. This skeleton project contains [licensing -information](LICENSE), as well as [pre-commit -hooks](https://pre-commit.com) and a [Travis -CI](https://travis-ci.com) configuration appropriate for a Python -library project. +information](LICENSE), as well as +[pre-commit hooks](https://pre-commit.com) and +[GitHub Actions](https://github.com/features/actions) configurations +appropriate for a Python library project. ## Contributing ## From e666a6fe9ff33bfbf944eab62841806ec2a983e1 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 17 Oct 2019 17:05:06 -0400 Subject: [PATCH 066/282] Add comment about why we are pinning virtualenv --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2100efe..4bac201 100644 --- a/setup.py +++ b/setup.py @@ -74,7 +74,7 @@ def package_vars(version_file): "pytest", "pytest-cov", "coveralls", - "virtualenv==16.3.0", + "virtualenv==16.3.0", # see: https://github.com/ansible/ansible-lint/issues/590 ] }, # Conveniently allows one to run the CLI tool as `example` From e4127dd5a82387edb6ff25e8812f7d91935360d1 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 17 Oct 2019 17:05:54 -0400 Subject: [PATCH 067/282] Rename env variable to make sense in a post-travis world. --- tests/test_example.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_example.py b/tests/test_example.py index 9b9ec4f..5e41e31 100644 --- a/tests/test_example.py +++ b/tests/test_example.py @@ -28,7 +28,7 @@ ) # define sources of version strings -TRAVIS_TAG = os.getenv("TRAVIS_TAG") +RELEASE_TAG = os.getenv("RELEASE_TAG") PROJECT_VERSION = example.__version__ @@ -44,13 +44,13 @@ def test_stdout_version(capsys): @pytest.mark.skipif( - TRAVIS_TAG in [None, ""], reason="this is not a release (TRAVIS_TAG not set)" + RELEASE_TAG in [None, ""], reason="this is not a release (RELEASE_TAG not set)" ) def test_release_version(): """Verify that release tag version agrees with the module version.""" assert ( - TRAVIS_TAG == f"v{PROJECT_VERSION}" - ), "TRAVIS_TAG does not match the project version" + RELEASE_TAG == f"v{PROJECT_VERSION}" + ), "RELEASE_TAG does not match the project version" @pytest.mark.parametrize("level", log_levels) From 038a75ad6f21205beabfcfc47ae282d4d1b46158 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 17 Oct 2019 17:06:09 -0400 Subject: [PATCH 068/282] Formatting --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b02ea85..507ab7e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,6 +10,7 @@ jobs: steps: - uses: actions/checkout@v1 + - name: Set up Python 3.7 uses: actions/setup-python@v1 with: From 78a230d46c532176c0aa1af70b33f3bca4664243 Mon Sep 17 00:00:00 2001 From: Dave Redmin Date: Fri, 18 Oct 2019 10:10:19 -0400 Subject: [PATCH 069/282] Add updates from running pre-commit autoupdate --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9580afa..0057d09 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,7 +29,7 @@ repos: args: - --config=.mdl_config.json - repo: https://github.com/adrienverge/yamllint - rev: v1.17.0 + rev: v1.18.0 hooks: - id: yamllint - repo: https://github.com/detailyang/pre-commit-shell @@ -43,7 +43,7 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.23.0 + rev: v1.25.0 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit From 05a136bb117b9fb966e0ce3bd0d9784a32f08c51 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 18 Oct 2019 10:39:22 -0400 Subject: [PATCH 070/282] Replace Travis-CI with GitHub actions. --- .github/workflows/build.yml | 26 ++++++++++++++++++++++++++ .travis.yml | 17 ----------------- README.md | 8 ++++---- 3 files changed, 30 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/build.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..b3e5b2f --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,26 @@ +--- +name: build + +on: [push] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade -r requirements-test.txt + + - name: Run pre-commit on all files + run: pre-commit run --all-files + diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 2c70e21..0000000 --- a/.travis.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -dist: xenial -language: python -python: 3.7 -# pre-commit hooks can use Docker, so we should go ahead and enable it -services: docker - -# Cache pip packages and pre-commit plugins to speed up builds -cache: - pip: true - directories: - - $HOME/.cache/pre-commit - -install: - - pip install --upgrade -r requirements-test.txt -script: - - pre-commit run --all-files diff --git a/README.md b/README.md index b880f4d..36c7336 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ # skeleton-generic # -[![Build Status](https://travis-ci.com/cisagov/skeleton-generic.svg?branch=develop)](https://travis-ci.com/cisagov/skeleton-generic) +[![GitHub Build Status](https://github.com/cisagov/skeleton-generic/workflows/build/badge.svg)](https://github.com/cisagov/skeleton-python-library/actions) This is a generic skeleton project that can be used to quickly get a new [cisagov](https://github.com/cisagov) GitHub project started. This skeleton project contains [licensing information](LICENSE), as -well as [pre-commit hooks](https://pre-commit.com) and a [Travis -CI](https://travis-ci.com) configuration appropriate for the major -languages that we use. +well as [pre-commit hooks](https://pre-commit.com) and +[GitHub Actions](https://github.com/features/actions) configurations +appropriate for the major languages that we use. In many cases you will instead want to use one of the more specific skeleton projects derived from this one. From 02fb008d62ca07a78a0103a7af1d81b4712613c1 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 18 Oct 2019 10:40:58 -0400 Subject: [PATCH 071/282] Fix action shield link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 36c7336..606df0a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # skeleton-generic # -[![GitHub Build Status](https://github.com/cisagov/skeleton-generic/workflows/build/badge.svg)](https://github.com/cisagov/skeleton-python-library/actions) +[![GitHub Build Status](https://github.com/cisagov/skeleton-generic/workflows/build/badge.svg)](https://github.com/cisagov/skeleton-generic/actions) This is a generic skeleton project that can be used to quickly get a new [cisagov](https://github.com/cisagov) GitHub project started. From b31ad8591f030b2300b82775dfce2fb3c85fc8db Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 18 Oct 2019 10:46:32 -0400 Subject: [PATCH 072/282] Disable ansible-lint until they fix their setup.py See: https://github.com/ansible/ansible-lint/issues/590 --- .pre-commit-config.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0057d09..fca1e26 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -56,11 +56,12 @@ repos: rev: 19.3b0 hooks: - id: black - - repo: https://github.com/ansible/ansible-lint.git - rev: v4.1.1a0 - hooks: - - id: ansible-lint - # files: molecule/default/playbook.yml + # Disabled until: https://github.com/ansible/ansible-lint/issues/590 + # - repo: https://github.com/ansible/ansible-lint.git + # rev: v4.1.1a0 + # hooks: + # - id: ansible-lint + # # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git rev: v1.12.0 hooks: From b11e39abd7e62b93f9a14158919bd524317ff34e Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 18 Oct 2019 11:00:39 -0400 Subject: [PATCH 073/282] Remove extra line. --- .github/workflows/build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b3e5b2f..013abd4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,4 +23,3 @@ jobs: - name: Run pre-commit on all files run: pre-commit run --all-files - From df0d9220a5aada87078fc4a0f4c110274c4880f9 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 25 Oct 2019 14:36:23 -0400 Subject: [PATCH 074/282] Split workflow into multiple jobs. Add artifact output. --- .github/workflows/build.yml | 48 +++++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 507ab7e..55faac1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,31 +4,53 @@ name: build on: [push] jobs: - build: - + lint: runs-on: ubuntu-latest - steps: - uses: actions/checkout@v1 - - - name: Set up Python 3.7 - uses: actions/setup-python@v1 + - uses: actions/setup-python@v1 with: python-version: 3.7 - - name: Install dependencies run: | python -m pip install --upgrade pip pip install --upgrade -r requirements-test.txt - - - name: Run pre-commit on all files + - name: Run linters on all files run: pre-commit run --all-files - - - name: Test with pytest + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade -r requirements-test.txt + - name: Run tests run: pytest - - - name: Coveralls + - name: Upload coverage report run: coveralls env: COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} if: success() + build: + runs-on: ubuntu-latest + needs: [lint, test] + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Install dependencies + run: | + python -m pip install --upgrade pip wheel + pip install --upgrade -r requirements.txt + - name: Build artifacts + run: python3 setup.py sdist bdist_wheel + - name: Upload artifacts + uses: actions/upload-artifact@v1 + with: + name: dist + path: dist From 3c78dd98a6ee54b1a102f4e4bddb6c0a31cf4c1d Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sat, 2 Nov 2019 19:48:42 -0400 Subject: [PATCH 075/282] Re-enabled ansible-lint and ran pre-commit autoupgrade. --- .pre-commit-config.yaml | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fca1e26..6d7bcbc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ --- repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.3.0 + rev: v2.4.0 hooks: - id: check-executables-have-shebangs - id: check-json @@ -23,7 +23,7 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.18.0 + rev: v0.19.0 hooks: - id: markdownlint args: @@ -37,13 +37,13 @@ repos: hooks: - id: shell-lint - repo: https://gitlab.com/pycqa/flake8 - rev: 3.7.8 + rev: 3.7.9 hooks: - id: flake8 additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.25.0 + rev: v1.25.1 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit @@ -53,15 +53,14 @@ repos: args: - --config=.bandit.yml - repo: https://github.com/python/black - rev: 19.3b0 + rev: 19.10b0 hooks: - id: black - # Disabled until: https://github.com/ansible/ansible-lint/issues/590 - # - repo: https://github.com/ansible/ansible-lint.git - # rev: v4.1.1a0 - # hooks: - # - id: ansible-lint - # # files: molecule/default/playbook.yml + - repo: https://github.com/ansible/ansible-lint.git + rev: v4.1.1a3 + hooks: + - id: ansible-lint + # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git rev: v1.12.0 hooks: From af4e9fe9f2d325de61d4b43ca3cd48bc2b775516 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 5 Nov 2019 12:07:00 -0500 Subject: [PATCH 076/282] Pull upstream changes from skeleton-generic. --- .pre-commit-config.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e8b13fc..51ef1aa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ --- repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.3.0 + rev: v2.4.0 hooks: - id: check-executables-have-shebangs - id: check-json @@ -23,7 +23,7 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.18.0 + rev: v0.19.0 hooks: - id: markdownlint args: @@ -37,13 +37,13 @@ repos: hooks: - id: shell-lint - repo: https://gitlab.com/pycqa/flake8 - rev: 3.7.8 + rev: 3.7.9 hooks: - id: flake8 additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.25.0 + rev: v1.25.1 hooks: - id: pyupgrade # Run bandit on "tests" tree with a configuration @@ -63,11 +63,11 @@ repos: name: bandit (everything else) exclude: tests - repo: https://github.com/python/black - rev: 19.3b0 + rev: 19.10b0 hooks: - id: black - repo: https://github.com/ansible/ansible-lint.git - rev: v4.1.1a0 + rev: v4.1.1a3 hooks: - id: ansible-lint # files: molecule/default/playbook.yml From 36f1b3bc9123581ed78c892578cf892b1b73bffa Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 5 Nov 2019 12:27:52 -0500 Subject: [PATCH 077/282] Remove virtualenv version pinning as the issue has been resolved in ansible-lint and the updated version is included in the version being used now. --- setup.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/setup.py b/setup.py index 4bac201..eb246c4 100644 --- a/setup.py +++ b/setup.py @@ -68,15 +68,7 @@ def package_vars(version_file): py_modules=[splitext(basename(path))[0] for path in glob("src/*.py")], include_package_data=True, install_requires=["docopt", "setuptools"], - extras_require={ - "test": [ - "pre-commit", - "pytest", - "pytest-cov", - "coveralls", - "virtualenv==16.3.0", # see: https://github.com/ansible/ansible-lint/issues/590 - ] - }, + extras_require={"test": ["pre-commit", "pytest", "pytest-cov", "coveralls"]}, # Conveniently allows one to run the CLI tool as `example` entry_points={"console_scripts": ["example = example.example:main"]}, ) From 5d5567d303c8ef4c1074ba86bfc7a73dad63b4c8 Mon Sep 17 00:00:00 2001 From: Felddy Date: Tue, 5 Nov 2019 14:27:20 -0500 Subject: [PATCH 078/282] Add GitHub action caching of pre-commit hooks and pip packages. --- .github/workflows/build.yml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 013abd4..fe47599 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,9 +5,7 @@ on: [push] jobs: build: - runs-on: ubuntu-latest - steps: - uses: actions/checkout@v1 @@ -16,6 +14,25 @@ jobs: with: python-version: 3.7 + - name: Cache pre-commit hooks + uses: actions/cache@v1 + with: + path: ~/.cache/pre-commit + key: "${{ runner.os }}-pre-commit-\ + ${{ hashFiles('**/.pre-commit-config.yaml') }}" + restore-keys: | + ${{ runner.os }}-pre-commit- + + - name: Cache pip test requirements + uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: "${{ runner.os }}-pip-test-\ + ${{ hashFiles('**/requirements-test.txt') }}" + restore-keys: | + ${{ runner.os }}-pip-test- + ${{ runner.os }}-pip- + - name: Install dependencies run: | python -m pip install --upgrade pip From 99c611557e432fe6615f7495b6dbdf5c78ac9ae6 Mon Sep 17 00:00:00 2001 From: Felddy Date: Tue, 5 Nov 2019 14:51:49 -0500 Subject: [PATCH 079/282] Remove search for previous pre-commit caches. These seem to be very large caches. Restoring an old one and updating it results in a cache larger than the maximum allowed cache size: 200MB. "Cache size of 254757924 bytes is over the 200MB limit, not saving cache." So if the config changes it is best to just take the cache-miss and start from scratch. --- .github/workflows/build.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fe47599..d12843d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,8 +20,6 @@ jobs: path: ~/.cache/pre-commit key: "${{ runner.os }}-pre-commit-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" - restore-keys: | - ${{ runner.os }}-pre-commit- - name: Cache pip test requirements uses: actions/cache@v1 From cf99067e441ef97565481ac8e96b2f73d6250bdb Mon Sep 17 00:00:00 2001 From: Felddy Date: Tue, 12 Nov 2019 10:10:27 -0500 Subject: [PATCH 080/282] Add caches for GitHub Actions. --- .github/workflows/build.yml | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 55faac1..fb06afa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,6 +3,10 @@ name: build on: [push] +env: + PIP_CACHE_DIR: ~/.cache/pip + PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit + jobs: lint: runs-on: ubuntu-latest @@ -11,6 +15,21 @@ jobs: - uses: actions/setup-python@v1 with: python-version: 3.7 + - name: Cache pip test requirements + uses: actions/cache@v1 + with: + path: ${{ env.PIP_CACHE_DIR }} + key: "${{ runner.os }}-pip-test-\ + ${{ hashFiles('**/requirements-test.txt') }}" + restore-keys: | + ${{ runner.os }}-pip-test- + ${{ runner.os }}-pip- + - name: Cache pre-commit hooks + uses: actions/cache@v1 + with: + path: ${{ env.PRE_COMMIT_CACHE_DIR }} + key: "${{ runner.os }}-pre-commit-\ + ${{ hashFiles('**/.pre-commit-config.yaml') }}" - name: Install dependencies run: | python -m pip install --upgrade pip @@ -24,6 +43,15 @@ jobs: - uses: actions/setup-python@v1 with: python-version: 3.7 + - name: Cache pip test requirements + uses: actions/cache@v1 + with: + path: ${{ env.PIP_CACHE_DIR }} + key: "${{ runner.os }}-pip-test-\ + ${{ hashFiles('**/requirements-test.txt') }}" + restore-keys: | + ${{ runner.os }}-pip-test- + ${{ runner.os }}-pip- - name: Install dependencies run: | python -m pip install --upgrade pip @@ -43,6 +71,15 @@ jobs: - uses: actions/setup-python@v1 with: python-version: 3.7 + - name: Cache pip build requirements + uses: actions/cache@v1 + with: + path: ${{ env.PIP_CACHE_DIR }} + key: "${{ runner.os }}-pip-build-\ + ${{ hashFiles('**/requirements.txt') }}" + restore-keys: | + ${{ runner.os }}-pip-build- + ${{ runner.os }}-pip- - name: Install dependencies run: | python -m pip install --upgrade pip wheel From b10dd19eb4bd84f6705d5d89b889061ce0a24528 Mon Sep 17 00:00:00 2001 From: Felddy Date: Tue, 12 Nov 2019 11:44:44 -0500 Subject: [PATCH 081/282] Pass in the github release tag for testing. --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fb06afa..e2c1a1f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -57,6 +57,8 @@ jobs: python -m pip install --upgrade pip pip install --upgrade -r requirements-test.txt - name: Run tests + env: + RELEASE_TAG: ${{ github.event.release.tag_name }} run: pytest - name: Upload coverage report run: coveralls From 43c35af94e9956fb9fd2b45fd193ba68b94c822f Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 14 Nov 2019 15:14:27 -0500 Subject: [PATCH 082/282] Applying autoupdate of pre-commit hooks. Just trying to tickle GitHub Actions so my tests will restart. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 51ef1aa..e6423b8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -67,7 +67,7 @@ repos: hooks: - id: black - repo: https://github.com/ansible/ansible-lint.git - rev: v4.1.1a3 + rev: v4.1.1a5 hooks: - id: ansible-lint # files: molecule/default/playbook.yml @@ -81,6 +81,6 @@ repos: hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier - rev: 1.18.2 + rev: 1.19.1 hooks: - id: prettier From 7c0b30b495e689b84317b390eff36425d7c70cdd Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 15 Nov 2019 11:50:38 -0500 Subject: [PATCH 083/282] Force pre-commit hooks to use python3. Fixes #23 --- .pre-commit-config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6d7bcbc..7626c79 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,8 @@ --- +default_language_version: + # force all unspecified python hooks to run python3 + python: python3 + repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v2.4.0 From 30a16a38420cca91cda0cd2543484aa4a2e125cc Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 15 Nov 2019 11:51:46 -0500 Subject: [PATCH 084/282] Autoupdate pre-commit hooks. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7626c79..27477d3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -61,7 +61,7 @@ repos: hooks: - id: black - repo: https://github.com/ansible/ansible-lint.git - rev: v4.1.1a3 + rev: v4.1.1a5 hooks: - id: ansible-lint # files: molecule/default/playbook.yml @@ -75,6 +75,6 @@ repos: hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier - rev: 1.18.2 + rev: 1.19.1 hooks: - id: prettier From e39dc8881915f36fdb70c9c58f0278700523b294 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 18 Nov 2019 10:33:28 -0500 Subject: [PATCH 085/282] Add codeowners file with team OIS maintainers. See: https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners --- .github/CODEOWNERS | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..ca9c8dc --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,7 @@ +# Each line is a file pattern followed by one or more owners. + +# These owners will be the default owners for everything in +# the repo. Unless a later match takes precedence, +# these owners will be requested for review when someone +# opens a pull request. +* @dav3r @felddy @jsf9k @mcdonnnj From b89eecb788084c95930554679cb38b32548c3c8d Mon Sep 17 00:00:00 2001 From: Mark Feldhousen Date: Mon, 18 Nov 2019 11:11:06 -0500 Subject: [PATCH 086/282] Update CODEOWNERS --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ca9c8dc..32918e8 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,4 +4,4 @@ # the repo. Unless a later match takes precedence, # these owners will be requested for review when someone # opens a pull request. -* @dav3r @felddy @jsf9k @mcdonnnj +* @dav3r @felddy @jsf9k @mcdonnnj @cisagov/team-ois From 18b5d12c784dad59c461af36922cc2082f1dcffd Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 19 Dec 2019 15:32:36 -0500 Subject: [PATCH 087/282] Add seed-isort-config and isort to pre-commit hooks. Add isort configuration file. --- .isort.cfg | 12 ++++++++++++ .pre-commit-config.yaml | 10 ++++++++++ 2 files changed, 22 insertions(+) create mode 100644 .isort.cfg diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 0000000..5216e47 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,12 @@ +[settings] +combine_star=true +force_sort_within_sections=true + +import_heading_stdlib=Standard Python Libraries +import_heading_thirdparty=Third-Party Libraries +import_heading_firstparty=cisagov Libraries + +# Should be auto-populated by seed-isort-config hook +known_third_party= +# These must be manually set to correctly separate them from third party libraries +known_first_party= diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 27477d3..c8fc88a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -60,6 +60,16 @@ repos: rev: 19.10b0 hooks: - id: black + - repo: https://github.com/asottile/seed-isort-config + rev: v1.9.3 + hooks: + - id: seed-isort-config + - repo: https://github.com/pre-commit/mirrors-isort + # pick the isort version you'd like to use from + # https://github.com/pre-commit/mirrors-isort/releases + rev: v4.3.21 + hooks: + - id: isort - repo: https://github.com/ansible/ansible-lint.git rev: v4.1.1a5 hooks: From db3e5c985422cf560249f7a169bda58b68ce9b7f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 23 Dec 2019 12:13:35 -0500 Subject: [PATCH 088/282] Ran pre-commit to populate .isort.cfg and let isort run against the repository. --- .isort.cfg | 4 ++-- setup.py | 6 ++++-- src/example/__init__.py | 2 +- src/example/example.py | 2 ++ tests/conftest.py | 1 + tests/test_example.py | 3 +++ 6 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index 5216e47..c0d5626 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -7,6 +7,6 @@ import_heading_thirdparty=Third-Party Libraries import_heading_firstparty=cisagov Libraries # Should be auto-populated by seed-isort-config hook -known_third_party= +known_third_party=docopt,pkg_resources,pytest,setuptools # These must be manually set to correctly separate them from third party libraries -known_first_party= +known_first_party=example diff --git a/setup.py b/setup.py index eb246c4..cc14932 100644 --- a/setup.py +++ b/setup.py @@ -8,10 +8,12 @@ - https://blog.ionelmc.ro/2014/05/25/python-packaging/#the-structure """ +# Standard Python Libraries from glob import glob -from os.path import splitext, basename +from os.path import basename, splitext -from setuptools import setup, find_packages +# Third-Party Libraries +from setuptools import find_packages, setup def readme(): diff --git a/src/example/__init__.py b/src/example/__init__.py index ba27a7c..cf807c3 100644 --- a/src/example/__init__.py +++ b/src/example/__init__.py @@ -1,5 +1,5 @@ """The example library.""" -from .example import example_div from ._version import __version__ # noqa: F401 +from .example import example_div __all__ = ["example_div"] diff --git a/src/example/example.py b/src/example/example.py index d389e0d..28b99f2 100755 --- a/src/example/example.py +++ b/src/example/example.py @@ -13,10 +13,12 @@ "warning", "error", and "critical". [default: warning] """ +# Standard Python Libraries import logging import os import sys +# Third-Party Libraries import docopt import pkg_resources diff --git a/tests/conftest.py b/tests/conftest.py index 58c399f..39e2be2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,7 @@ https://docs.pytest.org/en/latest/writing_plugins.html#conftest-py-plugins """ +# Third-Party Libraries import pytest diff --git a/tests/test_example.py b/tests/test_example.py index 5e41e31..4d1cef1 100644 --- a/tests/test_example.py +++ b/tests/test_example.py @@ -1,13 +1,16 @@ #!/usr/bin/env pytest -vs """Tests for example.""" +# Standard Python Libraries import logging import os import sys from unittest.mock import patch +# Third-Party Libraries import pytest +# cisagov Libraries import example div_params = [ From db2a01a0db0de0f840d5f33f4d4a3a69033edba4 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 23 Dec 2019 12:15:34 -0500 Subject: [PATCH 089/282] Add "coverage < 5.0" to setup.py to enforce it because coveralls does not currently support coverage 5.0+ per https://github.com/coveralls-clients/coveralls-python/issues/203 --- setup.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index cc14932..9223aac 100644 --- a/setup.py +++ b/setup.py @@ -70,7 +70,18 @@ def package_vars(version_file): py_modules=[splitext(basename(path))[0] for path in glob("src/*.py")], include_package_data=True, install_requires=["docopt", "setuptools"], - extras_require={"test": ["pre-commit", "pytest", "pytest-cov", "coveralls"]}, + extras_require={ + "test": [ + "pre-commit", + "coveralls", + # coveralls does not currently support coverage 5.0 + # https://github.com/coveralls-clients/coveralls-python/issues/203 + # is the issue for this on the coveralls project + "coverage < 5.0", + "pytest-cov", + "pytest", + ] + }, # Conveniently allows one to run the CLI tool as `example` entry_points={"console_scripts": ["example = example.example:main"]}, ) From 67c6960617deb52c8bc213c86698cdcedec3c403 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 10 Jan 2020 10:59:33 -0500 Subject: [PATCH 090/282] Add bump_version.sh and tag.sh to support bumping versions and tagging through scripts. Update setup.py to include the semver package which is used by the bump_version.sh script. --- bump_version.sh | 47 +++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 2 +- tag.sh | 9 +++++++++ 3 files changed, 57 insertions(+), 1 deletion(-) create mode 100755 bump_version.sh create mode 100755 tag.sh diff --git a/bump_version.sh b/bump_version.sh new file mode 100755 index 0000000..ed8f4e0 --- /dev/null +++ b/bump_version.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +# bump_version.sh (show|major|minor|patch|prerelease|build) + +set -o nounset +set -o errexit +set -o pipefail + +VERSION_FILE=src/example/_version.py + +HELP_INFORMATION="bump_version.sh (show|major|minor|patch|prerelease|build|finalize)" + +old_version=$(sed -n "s/^__version__ = \"\(.*\)\"$/\1/p" $VERSION_FILE) + +if [ $# -ne 1 ] +then + echo "$HELP_INFORMATION" +else + case $1 in + major|minor|patch|prerelease|build) + new_version=$(python -c "import semver; print(semver.bump_$1('$old_version'))") + echo Changing version from "$old_version" to "$new_version" + tmp_file=/tmp/version.$$ + sed "s/$old_version/$new_version/" $VERSION_FILE > $tmp_file + mv $tmp_file $VERSION_FILE + git add $VERSION_FILE + git commit -m"Bump version from $old_version to $new_version" + git push + ;; + finalize) + new_version=$(python -c "import semver; print(semver.finalize_version('$old_version'))") + echo Changing version from "$old_version" to "$new_version" + tmp_file=/tmp/version.$$ + sed "s/$old_version/$new_version/" $VERSION_FILE > $tmp_file + mv $tmp_file $VERSION_FILE + git add $VERSION_FILE + git commit -m"Bump version from $old_version to $new_version" + git push + ;; + show) + echo "$old_version" + ;; + *) + echo "$HELP_INFORMATION" + ;; + esac +fi diff --git a/setup.py b/setup.py index 9223aac..f534bf6 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,7 @@ def package_vars(version_file): package_data={"example": ["data/*.txt"]}, py_modules=[splitext(basename(path))[0] for path in glob("src/*.py")], include_package_data=True, - install_requires=["docopt", "setuptools"], + install_requires=["docopt", "semver", "setuptools"], extras_require={ "test": [ "pre-commit", diff --git a/tag.sh b/tag.sh new file mode 100755 index 0000000..e1f7447 --- /dev/null +++ b/tag.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -o nounset +set -o errexit +set -o pipefail + +version=$(./bump_version.sh show) + +git tag "v$version" && git push --tags From 5b6392d1b310b6c5f218ffcbe9ddd2c2b9222e93 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 10 Jan 2020 11:09:29 -0500 Subject: [PATCH 091/282] Move semver requirement to test from install. --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f534bf6..e5bfa9f 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,7 @@ def package_vars(version_file): package_data={"example": ["data/*.txt"]}, py_modules=[splitext(basename(path))[0] for path in glob("src/*.py")], include_package_data=True, - install_requires=["docopt", "semver", "setuptools"], + install_requires=["docopt", "setuptools"], extras_require={ "test": [ "pre-commit", @@ -80,6 +80,7 @@ def package_vars(version_file): "coverage < 5.0", "pytest-cov", "pytest", + "semver", ] }, # Conveniently allows one to run the CLI tool as `example` From e3e3d908b2a4222a6b7e3e920b104f5d4792771d Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 10 Jan 2020 12:21:04 -0500 Subject: [PATCH 092/282] Move semver requirement from test requirements in setup.py to requirements-dev.txt file to better match intent. --- requirements-dev.txt | 1 + setup.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index f122cc5..efe5c72 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,3 @@ -r requirements-test.txt ipython +semver diff --git a/setup.py b/setup.py index e5bfa9f..9223aac 100644 --- a/setup.py +++ b/setup.py @@ -80,7 +80,6 @@ def package_vars(version_file): "coverage < 5.0", "pytest-cov", "pytest", - "semver", ] }, # Conveniently allows one to run the CLI tool as `example` From 03278cab67fbb6965a0e7729609fdf33523dde61 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 10 Jan 2020 13:05:50 -0500 Subject: [PATCH 093/282] Add comments explaining why a temp file is used when updating the version in the bump_version.sh script. --- bump_version.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bump_version.sh b/bump_version.sh index ed8f4e0..861eed0 100755 --- a/bump_version.sh +++ b/bump_version.sh @@ -20,6 +20,8 @@ else major|minor|patch|prerelease|build) new_version=$(python -c "import semver; print(semver.bump_$1('$old_version'))") echo Changing version from "$old_version" to "$new_version" + # A temp file is used to provide compatability with macOS development + # as a result of macOS using the BSD version of sed tmp_file=/tmp/version.$$ sed "s/$old_version/$new_version/" $VERSION_FILE > $tmp_file mv $tmp_file $VERSION_FILE @@ -30,6 +32,8 @@ else finalize) new_version=$(python -c "import semver; print(semver.finalize_version('$old_version'))") echo Changing version from "$old_version" to "$new_version" + # A temp file is used to provide compatability with macOS development + # as a result of macOS using the BSD version of sed tmp_file=/tmp/version.$$ sed "s/$old_version/$new_version/" $VERSION_FILE > $tmp_file mv $tmp_file $VERSION_FILE From 92737fa2595578654e0e625cd45b9f522931a438 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 13 Jan 2020 17:39:06 -0500 Subject: [PATCH 094/282] Add python_requires corresponding to listed Python versions in setup.py Add versioning for setuptools to support inclusiong of python_requires argument. Note: This will require pip versions >=9.0.0 for support. --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9223aac..5f82b0e 100644 --- a/setup.py +++ b/setup.py @@ -62,6 +62,7 @@ def package_vars(version_file): "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", ], + python_requires=">=3.6", # What does your project relate to? keywords="skeleton", packages=find_packages(where="src"), @@ -69,7 +70,7 @@ def package_vars(version_file): package_data={"example": ["data/*.txt"]}, py_modules=[splitext(basename(path))[0] for path in glob("src/*.py")], include_package_data=True, - install_requires=["docopt", "setuptools"], + install_requires=["docopt", "setuptools >= 24.2.0"], extras_require={ "test": [ "pre-commit", From 2fa4cbe194ac05ff89e377ad91adfa3fc7eddb6f Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 24 Jan 2020 16:45:48 -0500 Subject: [PATCH 095/282] Make workflow run when a PR is opened, synchronized, or reopened A user forked cisagov/scan-target-data and created a pull request, but the required GitHub Action(s) did not run. This is presumably because the user does not have Actions enabled in his or her fork. Ideally, the required Action(s) would run in cisagov/scan-target-data when a PR to merge changes back is created. Based on my reading of this link, adding the "pull_request" event type should make this happen: https://help.github.com/en/actions/automating-your-workflow-with-github-actions/events-that-trigger-workflows#pull-request-events-for-forked-repositories --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d12843d..c9cc05b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,7 +1,7 @@ --- name: build -on: [push] +on: [push, pull_request] jobs: build: From dbd589d2e27b517726fec167987058b3d687992f Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 24 Jan 2020 17:10:49 -0500 Subject: [PATCH 096/282] Improve list formatting --- .github/workflows/build.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c9cc05b..e6c14e6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,7 +1,10 @@ --- name: build -on: [push, pull_request] +on: [ + push, + pull_request +] jobs: build: From aaef9088f0b3bee886d2c1fc09f4603987b1e785 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 27 Jan 2020 08:20:25 -0500 Subject: [PATCH 097/282] Pull in changes to build workflow from skeleton-generic to support correct workflow if someone forks and does a pull request. --- .github/workflows/build.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e2c1a1f..d56c763 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,7 +1,10 @@ --- name: build -on: [push] +on: [ + push, + pull_request +] env: PIP_CACHE_DIR: ~/.cache/pip From 7db87878bfd40ebe073e4860ba0fec3991c7a738 Mon Sep 17 00:00:00 2001 From: Felddy Date: Wed, 29 Jan 2020 16:50:39 -0500 Subject: [PATCH 098/282] Add schema library. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5f82b0e..d5839ce 100644 --- a/setup.py +++ b/setup.py @@ -70,7 +70,7 @@ def package_vars(version_file): package_data={"example": ["data/*.txt"]}, py_modules=[splitext(basename(path))[0] for path in glob("src/*.py")], include_package_data=True, - install_requires=["docopt", "setuptools >= 24.2.0"], + install_requires=["docopt", "setuptools >= 24.2.0", "schema"], extras_require={ "test": [ "pre-commit", From 229da937e2b982e9361de3b18893c23b65be84f9 Mon Sep 17 00:00:00 2001 From: Felddy Date: Wed, 29 Jan 2020 16:53:02 -0500 Subject: [PATCH 099/282] Add schema checks for arguments. Log output instead of print. Improve docs. Do some interesting parsing. --- .isort.cfg | 2 +- src/example/example.py | 67 +++++++++++++++++++++++++++++++----------- 2 files changed, 51 insertions(+), 18 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index c0d5626..1156f7e 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -7,6 +7,6 @@ import_heading_thirdparty=Third-Party Libraries import_heading_firstparty=cisagov Libraries # Should be auto-populated by seed-isort-config hook -known_third_party=docopt,pkg_resources,pytest,setuptools +known_third_party=docopt,pkg_resources,pytest,schema,setuptools # These must be manually set to correctly separate them from third party libraries known_first_party=example diff --git a/src/example/example.py b/src/example/example.py index 28b99f2..b2856d9 100755 --- a/src/example/example.py +++ b/src/example/example.py @@ -2,15 +2,23 @@ """example is an example Python library and tool. +Divide one integer by another and log the result. Also log some information +from an environment variable and a package resource. + +EXIT STATUS + This utility exits with one of the following values: + 0 Calculation completed successfully. + >0 An error occurred. + Usage: - example [--log-level=LEVEL] + example [--log-level=LEVEL] example (-h | --help) Options: -h --help Show this message. --log-level=LEVEL If specified, then the log level will be set to the specified value. Valid values are "debug", "info", - "warning", "error", and "critical". [default: warning] + "warning", "error", and "critical". [default: info] """ # Standard Python Libraries @@ -21,43 +29,68 @@ # Third-Party Libraries import docopt import pkg_resources +from schema import And, Schema, SchemaError, Use from ._version import __version__ DEFAULT_ECHO_MESSAGE = "Hello World from the example default!" -def example_div(x, y): +def example_div(dividend, divisor): """Print some logging messages.""" logging.debug("This is a debug message") logging.info("This is an info message") logging.warning("This is a warning message") logging.error("This is an error message") logging.critical("This is a critical message") - return x / y + return dividend / divisor def main(): """Set up logging and call the example function.""" args = docopt.docopt(__doc__, version=__version__) - # Set up logging - log_level = args["--log-level"] + # Validate and convert arguments as needed + schema = Schema( + { + "--log-level": And( + str, + Use(str.lower), + lambda n: n in ("debug", "info", "warning", "error", "critical"), + error="Possible values for --log-level are " + + "debug, info, warning, error, and critical.", + ), + "": Use(int, error=" must be an integer."), + "": And( + Use(int), + lambda n: n != 0, + error=" must be an integer that is not 0.", + ), + str: object, # Don't care about other keys, if any + } + ) + try: - logging.basicConfig( - format="%(asctime)-15s %(levelname)s %(message)s", level=log_level.upper() - ) - except ValueError: - logging.critical( - f'"{log_level}" is not a valid logging level. Possible values ' - "are debug, info, warning, and error." - ) + args = schema.validate(args) + except SchemaError as err: + # Exit because one or more of the arguments were invalid + print(err, file=sys.stderr) return 1 - print(f"8 / 2 == {example_div(8, 2)}") + # Assign validated arguments to variables + dividend = args[""] + divisor = args[""] + log_level = args["--log-level"] + + # Set up logging + logging.basicConfig( + format="%(asctime)-15s %(levelname)s %(message)s", level=log_level.upper() + ) + + logging.info(f"{dividend} / {divisor} == {example_div(dividend, divisor)}") # Access some data from an environment variable message = os.getenv("ECHO_MESSAGE", DEFAULT_ECHO_MESSAGE) - print(f'ECHO_MESSAGE="{message}"') + logging.info(f'ECHO_MESSAGE="{message}"') # Access some data from our package data (see the setup.py) secret_message = ( @@ -65,7 +98,7 @@ def main(): .decode("utf-8") .strip() ) - print(f'Secret="{secret_message}"') + logging.info(f'Secret="{secret_message}"') # Stop logging and clean up logging.shutdown() From 7cbaaa922b1dafbc10bd3c0deb82070330e15a33 Mon Sep 17 00:00:00 2001 From: Felddy Date: Wed, 29 Jan 2020 16:54:37 -0500 Subject: [PATCH 100/282] Add tests for new arguments --- tests/test_example.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_example.py b/tests/test_example.py index 4d1cef1..dd10029 100644 --- a/tests/test_example.py +++ b/tests/test_example.py @@ -59,7 +59,7 @@ def test_release_version(): @pytest.mark.parametrize("level", log_levels) def test_log_levels(level): """Validate commandline log-level arguments.""" - with patch.object(sys, "argv", ["bogus", f"--log-level={level}"]): + with patch.object(sys, "argv", ["bogus", f"--log-level={level}", "1", "1"]): with patch.object(logging.root, "handlers", []): assert ( logging.root.hasHandlers() is False @@ -96,3 +96,10 @@ def test_zero_division(): """Verify that division by zero throws the correct exception.""" with pytest.raises(ZeroDivisionError): example.example_div(1, 0) + + +def test_zero_divisor_argument(): + """Verify that a divisor of zero is handled as expected.""" + with patch.object(sys, "argv", ["bogus", "1", "0"]): + return_code = example.example.main() + assert return_code == 1, "main() should exit with error" From 8520a7061e8161b71910fd069987b8deed9c54e6 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 Jan 2020 11:27:06 -0500 Subject: [PATCH 101/282] Removed uses of xfail. This was a bad example of when to use xfail. See the semantics here: http://doc.pytest.org/en/latest/skipping.html --- tests/test_example.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/test_example.py b/tests/test_example.py index dd10029..c3334d7 100644 --- a/tests/test_example.py +++ b/tests/test_example.py @@ -18,7 +18,6 @@ (2, 2, 1), (0, 1, 0), (8, 2, 4), - pytest.param(0, 0, 0, marks=pytest.mark.xfail(raises=ZeroDivisionError)), ] log_levels = ( @@ -27,7 +26,6 @@ "warning", "error", "critical", - pytest.param("critical2", marks=pytest.mark.xfail), ) # define sources of version strings @@ -71,6 +69,13 @@ def test_log_levels(level): assert return_code == 0, "main() should return success (0)" +def test_bad_log_level(): + """Validate bad log-level argument returns error.""" + with patch.object(sys, "argv", ["bogus", "--log-level=emergency", "1", "1"]): + return_code = example.example.main() + assert return_code == 1, "main() should return failure" + + @pytest.mark.parametrize("dividend, divisor, quotient", div_params) def test_division(dividend, divisor, quotient): """Verify division results.""" From 454553ed62207464a13e32c4454f75a3fa68c91b Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 Jan 2020 12:19:36 -0500 Subject: [PATCH 102/282] Bumping github actions python version to 3.8. Advertise support for 3.8 in setup.py --- .github/workflows/build.yml | 6 +++--- setup.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d56c763..60cc9b9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v1 - uses: actions/setup-python@v1 with: - python-version: 3.7 + python-version: 3.8 - name: Cache pip test requirements uses: actions/cache@v1 with: @@ -45,7 +45,7 @@ jobs: - uses: actions/checkout@v1 - uses: actions/setup-python@v1 with: - python-version: 3.7 + python-version: 3.8 - name: Cache pip test requirements uses: actions/cache@v1 with: @@ -75,7 +75,7 @@ jobs: - uses: actions/checkout@v1 - uses: actions/setup-python@v1 with: - python-version: 3.7 + python-version: 3.8 - name: Cache pip build requirements uses: actions/cache@v1 with: diff --git a/setup.py b/setup.py index d5839ce..ad59e1a 100644 --- a/setup.py +++ b/setup.py @@ -61,6 +61,7 @@ def package_vars(version_file): "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", ], python_requires=">=3.6", # What does your project relate to? From 532751627a347637814e8a7f626dad487a1b6053 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 10 Feb 2020 10:39:07 -0500 Subject: [PATCH 103/282] Backported changes to CONTRIBUTING.md from the development guide. --- CONTRIBUTING.md | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 93addc2..eb00ca9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -56,10 +56,31 @@ eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" ``` -For Linux (or on the Mac, if you don't want to use `brew`) you can use +For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you +don't want to use `brew`) you can use [pyenv/pyenv-installer](https://github.com/pyenv/pyenv-installer) to -install the necessary tools. When you are finished you will need to -add the same two lines above to your profile. +install the necessary tools. Before running this ensure that you have +installed the prerequisites for your platform according to the +[`pyenv` wiki +page](https://github.com/pyenv/pyenv/wiki/common-build-problems). + +On WSL you should treat your platform as whatever Linux distribution +you've chosen to install. + +Once you have installed `pyenv` you will need to add the following +lines to your `.bashrc`: + +```bash +export PATH="$PATH:$HOME/.pyenv/bin" +eval "$(pyenv init -)" +eval "$(pyenv virtualenv-init -)" +``` + +If you are using a shell other than `bash` you should follow the +instructions that the `pyenv-installer` script outputs. + +You will need to reload your shell for these changes to take effect so +you can begin to use `pyenv`. For a list of Python versions that are already installed and ready to use with `pyenv`, use the command `pyenv versions`. To see a list of From f7a4166ad67d961324bc44130e092eb1ddebd320 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 11 Feb 2020 10:41:17 -0500 Subject: [PATCH 104/282] Update Python version used to 3.8 Update actions/checkout to v2 Update formatting to match downstream children --- .github/workflows/build.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e6c14e6..4953f7c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,20 +10,16 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 - - - name: Set up Python 3.7 - uses: actions/setup-python@v1 + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 with: - python-version: 3.7 - + python-version: 3.8 - name: Cache pre-commit hooks uses: actions/cache@v1 with: path: ~/.cache/pre-commit key: "${{ runner.os }}-pre-commit-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" - - name: Cache pip test requirements uses: actions/cache@v1 with: @@ -33,11 +29,9 @@ jobs: restore-keys: | ${{ runner.os }}-pip-test- ${{ runner.os }}-pip- - - name: Install dependencies run: | python -m pip install --upgrade pip pip install --upgrade -r requirements-test.txt - - name: Run pre-commit on all files run: pre-commit run --all-files From b857939b3d0c5393b8b5528a6bb9bce2ff0f736f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 12 Feb 2020 00:01:16 -0500 Subject: [PATCH 105/282] Run pre-commit autoupdate. --- .pre-commit-config.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c8fc88a..7856658 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.4.0 + rev: v2.5.0 hooks: - id: check-executables-have-shebangs - id: check-json @@ -27,13 +27,13 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.19.0 + rev: v0.22.0 hooks: - id: markdownlint args: - --config=.mdl_config.json - repo: https://github.com/adrienverge/yamllint - rev: v1.18.0 + rev: v1.20.0 hooks: - id: yamllint - repo: https://github.com/detailyang/pre-commit-shell @@ -47,7 +47,7 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.25.1 + rev: v1.26.2 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit @@ -61,7 +61,7 @@ repos: hooks: - id: black - repo: https://github.com/asottile/seed-isort-config - rev: v1.9.3 + rev: v1.9.4 hooks: - id: seed-isort-config - repo: https://github.com/pre-commit/mirrors-isort @@ -71,7 +71,7 @@ repos: hooks: - id: isort - repo: https://github.com/ansible/ansible-lint.git - rev: v4.1.1a5 + rev: v4.2.0 hooks: - id: ansible-lint # files: molecule/default/playbook.yml @@ -81,7 +81,7 @@ repos: - id: terraform_fmt - id: terraform_validate_no_variables - repo: https://github.com/IamTheFij/docker-pre-commit - rev: v1.0.0 + rev: v1.0.1 hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier From d99fd00bc2e5c4a0afeb2d6717dac7fe77f64d33 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 12 Feb 2020 00:14:11 -0500 Subject: [PATCH 106/282] Flip cache order to mirror how it is done downstream. --- .github/workflows/build.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4953f7c..76801a8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,12 +14,6 @@ jobs: - uses: actions/setup-python@v1 with: python-version: 3.8 - - name: Cache pre-commit hooks - uses: actions/cache@v1 - with: - path: ~/.cache/pre-commit - key: "${{ runner.os }}-pre-commit-\ - ${{ hashFiles('**/.pre-commit-config.yaml') }}" - name: Cache pip test requirements uses: actions/cache@v1 with: @@ -29,6 +23,12 @@ jobs: restore-keys: | ${{ runner.os }}-pip-test- ${{ runner.os }}-pip- + - name: Cache pre-commit hooks + uses: actions/cache@v1 + with: + path: ~/.cache/pre-commit + key: "${{ runner.os }}-pre-commit-\ + ${{ hashFiles('**/.pre-commit-config.yaml') }}" - name: Install dependencies run: | python -m pip install --upgrade pip From fd033377fe93d865ab2cc98042a79c462dbbd976 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 12 Feb 2020 11:41:16 -0500 Subject: [PATCH 107/282] Fix spacing before comment in .bandit.yml (shown with pre-commit run --verbose --all-files). --- .bandit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bandit.yml b/.bandit.yml index 7b89269..0b53a96 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -11,4 +11,4 @@ tests: # - B102 skips: - - B101 # skip "assert used" check since assertions are required in pytests + - B101 # skip "assert used" check since assertions are required in pytests From 49f7002b6e3d185718ff78361e6b28e225a75787 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 12 Feb 2020 15:27:50 -0500 Subject: [PATCH 108/282] Unpin coverage as coveralls supports coverage 5.0+ as of 1.10.0 --- setup.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/setup.py b/setup.py index ad59e1a..928afe9 100644 --- a/setup.py +++ b/setup.py @@ -73,16 +73,7 @@ def package_vars(version_file): include_package_data=True, install_requires=["docopt", "setuptools >= 24.2.0", "schema"], extras_require={ - "test": [ - "pre-commit", - "coveralls", - # coveralls does not currently support coverage 5.0 - # https://github.com/coveralls-clients/coveralls-python/issues/203 - # is the issue for this on the coveralls project - "coverage < 5.0", - "pytest-cov", - "pytest", - ] + "test": ["pre-commit", "coveralls", "coverage", "pytest-cov", "pytest"] }, # Conveniently allows one to run the CLI tool as `example` entry_points={"console_scripts": ["example = example.example:main"]}, From 96675a15987dcf1ecdcecc8c9d2d6df0874507ef Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 12 Feb 2020 15:40:18 -0500 Subject: [PATCH 109/282] Add --verbose flag to the coveralls call in the test workflow. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4cb95ba..9afd8c3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -64,7 +64,7 @@ jobs: RELEASE_TAG: ${{ github.event.release.tag_name }} run: pytest - name: Upload coverage report - run: coveralls + run: coveralls --verbose env: COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} if: success() From df8a7eb68c72a9400b9b544d526d965a1a74b503 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 12 Feb 2020 15:56:23 -0500 Subject: [PATCH 110/282] Pin coveralls version to see if the service_number addition in 1.11.0 is breaking coveralls uploads. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 928afe9..b7bbe70 100644 --- a/setup.py +++ b/setup.py @@ -73,7 +73,7 @@ def package_vars(version_file): include_package_data=True, install_requires=["docopt", "setuptools >= 24.2.0", "schema"], extras_require={ - "test": ["pre-commit", "coveralls", "coverage", "pytest-cov", "pytest"] + "test": ["pre-commit", "coveralls < 1.11.0", "coverage", "pytest-cov", "pytest"] }, # Conveniently allows one to run the CLI tool as `example` entry_points={"console_scripts": ["example = example.example:main"]}, From 8785abe1d4e84a60fae2a551aab0096eafa169d9 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 12 Feb 2020 16:17:49 -0500 Subject: [PATCH 111/282] Remove --verbose from coveralls call in test workflow. Add comment explaining why coveralls is being pinned. --- .github/workflows/build.yml | 2 +- setup.py | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9afd8c3..4cb95ba 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -64,7 +64,7 @@ jobs: RELEASE_TAG: ${{ github.event.release.tag_name }} run: pytest - name: Upload coverage report - run: coveralls --verbose + run: coveralls env: COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} if: success() diff --git a/setup.py b/setup.py index b7bbe70..afda7e8 100644 --- a/setup.py +++ b/setup.py @@ -73,7 +73,19 @@ def package_vars(version_file): include_package_data=True, install_requires=["docopt", "setuptools >= 24.2.0", "schema"], extras_require={ - "test": ["pre-commit", "coveralls < 1.11.0", "coverage", "pytest-cov", "pytest"] + "test": [ + "pre-commit", + # coveralls 1.11.0 added a service number for calls from + # GitHub Actions. When run this resulted in a 422 response from the + # coveralls API with the message: + # Unprocessable Entity for url: https://coveralls.io/api/v1/jobs + # Temporarily pinning coveralls to before this version until this + # issue is resolved. + "coveralls < 1.11.0", + "coverage", + "pytest-cov", + "pytest", + ] }, # Conveniently allows one to run the CLI tool as `example` entry_points={"console_scripts": ["example = example.example:main"]}, From 6a7210cc3d2f0f976eec198f620d0ebfc338be57 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 14 Feb 2020 15:15:31 -0500 Subject: [PATCH 112/282] Pin to testing branch per https://github.com/coveralls-clients/coveralls-python/issues/219 for testing. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index afda7e8..e0d5003 100644 --- a/setup.py +++ b/setup.py @@ -81,7 +81,7 @@ def package_vars(version_file): # Unprocessable Entity for url: https://coveralls.io/api/v1/jobs # Temporarily pinning coveralls to before this version until this # issue is resolved. - "coveralls < 1.11.0", + "coveralls @ git+https://github.com/coveralls-clients/coveralls-python.git@fix-github-actions", "coverage", "pytest-cov", "pytest", From d59f5f97cdfee079a817263fae1d7f797941a77b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sun, 16 Feb 2020 15:38:30 -0500 Subject: [PATCH 113/282] Change coveralls pin to avoid the version with the regression. 1.11.1 is released which fixed the regression introduced in 1.11.0 per https://github.com/coveralls-clients/coveralls-python/releases/tag/1.11.1 --- setup.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index e0d5003..ac0d34e 100644 --- a/setup.py +++ b/setup.py @@ -76,12 +76,12 @@ def package_vars(version_file): "test": [ "pre-commit", # coveralls 1.11.0 added a service number for calls from - # GitHub Actions. When run this resulted in a 422 response from the - # coveralls API with the message: + # GitHub Actions. This caused a regression which resulted in a 422 + # response from the coveralls API with the message: # Unprocessable Entity for url: https://coveralls.io/api/v1/jobs - # Temporarily pinning coveralls to before this version until this - # issue is resolved. - "coveralls @ git+https://github.com/coveralls-clients/coveralls-python.git@fix-github-actions", + # 1.11.1 fixed this issue, but to ensure expected behavior we'll pin + # to never grab the regression version. + "coveralls != 1.11.0", "coverage", "pytest-cov", "pytest", From e96577bce4b3b6aefa044943e478301a7d11288f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 18 Feb 2020 18:04:06 -0500 Subject: [PATCH 114/282] All references to '-r' for pip calls have been replaced with the more verbose '--requirement'. --- .github/workflows/build.yml | 2 +- CONTRIBUTING.md | 2 +- requirements-dev.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 76801a8..aff7e7a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,6 +32,6 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install --upgrade -r requirements-test.txt + pip install --upgrade --requirement requirements-test.txt - name: Run pre-commit on all files run: pre-commit run --all-files diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index eb00ca9..dacaaad 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -102,7 +102,7 @@ commands: cd skeleton-generic pyenv virtualenv skeleton-generic pyenv local skeleton-generic -pip install -r requirements-dev.txt +pip install --requirement requirements-dev.txt ``` #### Installing the pre-commit hook #### diff --git a/requirements-dev.txt b/requirements-dev.txt index f122cc5..d84ee68 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,2 @@ --r requirements-test.txt +--requirement requirements-test.txt ipython From 0478f94b8cd74cbe0c6b24f4b53535a0a92bb280 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 19 Feb 2020 11:48:02 -0500 Subject: [PATCH 115/282] Adjust language in description of pre-commit step in the GitHub Actions workflow. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cd48651..09535e5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -37,7 +37,7 @@ jobs: run: | python -m pip install --upgrade pip pip install --upgrade --requirement requirements-test.txt - - name: Run linters on all files + - name: Run pre-commit on all files run: pre-commit run --all-files test: runs-on: ubuntu-latest From af261c328c1aefac59b422de973c59662094190d Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 20 Feb 2020 17:29:09 -0500 Subject: [PATCH 116/282] Autoupdate pre-commit hooks. Add mypy. --- .pre-commit-config.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1522593..df79e95 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -47,7 +47,7 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.26.2 + rev: v2.0.0 hooks: - id: pyupgrade # Run bandit on "tests" tree with a configuration @@ -84,7 +84,7 @@ repos: rev: v4.2.0 hooks: - id: ansible-lint - # files: molecule/default/playbook.yml + # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git rev: v1.12.0 hooks: @@ -98,3 +98,7 @@ repos: rev: 1.19.1 hooks: - id: prettier + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.761 + hooks: + - id: mypy From b68db4ce5f1c19327c38a37ad3e6ccb6ce32eb97 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 20 Feb 2020 17:29:58 -0500 Subject: [PATCH 117/282] Sort .gitignore add mypy cache. --- .gitignore | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 724760e..bedb6e4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ -*.egg-info __pycache__ -.python-version .coverage +.mypy_cache .pytest_cache +.python-version +*.egg-info From 3172a9924060b1bfb2b5c1d7edd0852154ba4a10 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 20 Feb 2020 17:35:43 -0500 Subject: [PATCH 118/282] Add PEP 484 type hints. See: https://www.python.org/dev/peps/pep-0484/ --- src/example/example.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/example/example.py b/src/example/example.py index b2856d9..556e8b5 100755 --- a/src/example/example.py +++ b/src/example/example.py @@ -25,6 +25,7 @@ import logging import os import sys +from typing import Any, Dict # Third-Party Libraries import docopt @@ -33,10 +34,10 @@ from ._version import __version__ -DEFAULT_ECHO_MESSAGE = "Hello World from the example default!" +DEFAULT_ECHO_MESSAGE: str = "Hello World from the example default!" -def example_div(dividend, divisor): +def example_div(dividend: float, divisor: float) -> float: """Print some logging messages.""" logging.debug("This is a debug message") logging.info("This is an info message") @@ -46,11 +47,11 @@ def example_div(dividend, divisor): return dividend / divisor -def main(): +def main() -> int: """Set up logging and call the example function.""" - args = docopt.docopt(__doc__, version=__version__) + args: Dict[str, str] = docopt.docopt(__doc__, version=__version__) # Validate and convert arguments as needed - schema = Schema( + schema: Schema = Schema( { "--log-level": And( str, @@ -70,16 +71,16 @@ def main(): ) try: - args = schema.validate(args) + validated_args: Dict[str, Any] = schema.validate(args) except SchemaError as err: # Exit because one or more of the arguments were invalid print(err, file=sys.stderr) return 1 # Assign validated arguments to variables - dividend = args[""] - divisor = args[""] - log_level = args["--log-level"] + dividend: int = validated_args[""] + divisor: int = validated_args[""] + log_level: str = validated_args["--log-level"] # Set up logging logging.basicConfig( @@ -89,11 +90,11 @@ def main(): logging.info(f"{dividend} / {divisor} == {example_div(dividend, divisor)}") # Access some data from an environment variable - message = os.getenv("ECHO_MESSAGE", DEFAULT_ECHO_MESSAGE) + message: str = os.getenv("ECHO_MESSAGE", DEFAULT_ECHO_MESSAGE) logging.info(f'ECHO_MESSAGE="{message}"') # Access some data from our package data (see the setup.py) - secret_message = ( + secret_message: str = ( pkg_resources.resource_string("example", "data/secret.txt") .decode("utf-8") .strip() From 067ee0850c154845b7de623988c5a1bd5ce67d3a Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 20 Feb 2020 17:29:09 -0500 Subject: [PATCH 119/282] Autoupdate pre-commit hooks. Add mypy. --- .pre-commit-config.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7856658..46cea9e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -47,7 +47,7 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.26.2 + rev: v2.0.0 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit @@ -74,7 +74,7 @@ repos: rev: v4.2.0 hooks: - id: ansible-lint - # files: molecule/default/playbook.yml + # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git rev: v1.12.0 hooks: @@ -88,3 +88,7 @@ repos: rev: 1.19.1 hooks: - id: prettier + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.761 + hooks: + - id: mypy From bf366086f48d9b7a7e49b9f44f33b44f6a23aeb3 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 2 Mar 2020 08:11:32 -0500 Subject: [PATCH 120/282] Add .mypy_cache to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 073a081..95b74cd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ +.mypy_cache __pycache__ .python-version From 454864bc029ce968cbeb8f9f5b88ad1c54522335 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 3 Mar 2020 16:49:53 -0500 Subject: [PATCH 121/282] Incorporate the Python version into keys for pip and pre-commit caches. This should resolve the issue seen when the Python version changes before there is an update to .pre-commit-config.yml which results in pre-commit pointing to a non-existent Python installation. --- .github/workflows/build.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index aff7e7a..a92cd83 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,20 +14,25 @@ jobs: - uses: actions/setup-python@v1 with: python-version: 3.8 + - name: Store installed Python version + run: | + echo "::set-env name=PY_VERSION::"\ + "$(python -c "import platform;print(platform.python_version())")" - name: Cache pip test requirements uses: actions/cache@v1 with: path: ~/.cache/pip - key: "${{ runner.os }}-pip-test-\ + key: "${{ runner.os }}-pip-test-py${{ env.PY_VERSION }}-\ ${{ hashFiles('**/requirements-test.txt') }}" restore-keys: | + ${{ runner.os }}-pip-test-py${{ env.PY_VERSION }}- ${{ runner.os }}-pip-test- ${{ runner.os }}-pip- - name: Cache pre-commit hooks uses: actions/cache@v1 with: path: ~/.cache/pre-commit - key: "${{ runner.os }}-pre-commit-\ + key: "${{ runner.os }}-pre-commit-py${{ env.PY_VERSION }}-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" - name: Install dependencies run: | From 478f1688a108dd0f6e634c5f03c7ba6cb816e3f2 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 4 Mar 2020 08:41:59 -0500 Subject: [PATCH 122/282] Change the cache paths from hardcoded values in their appropriate blocks to environment variables declared before the job block. --- .github/workflows/build.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a92cd83..298e93c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,6 +6,10 @@ on: [ pull_request ] +env: + PIP_CACHE_DIR: ~/.cache/pip + PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit + jobs: build: runs-on: ubuntu-latest @@ -21,7 +25,7 @@ jobs: - name: Cache pip test requirements uses: actions/cache@v1 with: - path: ~/.cache/pip + path: ${{ env.PIP_CACHE_DIR }} key: "${{ runner.os }}-pip-test-py${{ env.PY_VERSION }}-\ ${{ hashFiles('**/requirements-test.txt') }}" restore-keys: | @@ -31,7 +35,7 @@ jobs: - name: Cache pre-commit hooks uses: actions/cache@v1 with: - path: ~/.cache/pre-commit + path: ${{ env.PRE_COMMIT_CACHE_DIR }} key: "${{ runner.os }}-pre-commit-py${{ env.PY_VERSION }}-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" - name: Install dependencies From d7913343523e9841e1d295a5203e2043f055064b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 4 Mar 2020 10:56:39 -0500 Subject: [PATCH 123/282] Add a rule for markdownlint to allow multiple headers with the same name as long as they are not nested in the same heading group. --- .mdl_config.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.mdl_config.json b/.mdl_config.json index 492955a..7a6f3f8 100644 --- a/.mdl_config.json +++ b/.mdl_config.json @@ -3,5 +3,8 @@ "code_blocks": false, "tables": false }, + "MD024": { + "allow_different_nesting": true + }, "default": true } From 61790a9bbd22cede3cd41d94115d0e8ad8ba51d1 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 31 Mar 2020 14:41:10 -0400 Subject: [PATCH 124/282] Fixed broken hook id for pre-commit-terraform: terraform_validate_no_variables was changed to terraform_validate in the following commit: https://github.com/antonbabenko/pre-commit-terraform/commit/35e0356188b64a4c5af9a4e7200d936e514cba71. Ran pre-commit autoupdate. --- .pre-commit-config.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 46cea9e..42b824f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,7 +33,7 @@ repos: args: - --config=.mdl_config.json - repo: https://github.com/adrienverge/yamllint - rev: v1.20.0 + rev: v1.21.0 hooks: - id: yamllint - repo: https://github.com/detailyang/pre-commit-shell @@ -47,7 +47,7 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v2.0.0 + rev: v2.1.0 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit @@ -61,7 +61,7 @@ repos: hooks: - id: black - repo: https://github.com/asottile/seed-isort-config - rev: v1.9.4 + rev: v2.1.0 hooks: - id: seed-isort-config - repo: https://github.com/pre-commit/mirrors-isort @@ -76,19 +76,19 @@ repos: - id: ansible-lint # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git - rev: v1.12.0 + rev: v1.27.0 hooks: - id: terraform_fmt - - id: terraform_validate_no_variables + - id: terraform_validate - repo: https://github.com/IamTheFij/docker-pre-commit rev: v1.0.1 hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier - rev: 1.19.1 + rev: 2.0.2 hooks: - id: prettier - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.761 + rev: v0.770 hooks: - id: mypy From 0bc1aab45e2338e658a63a9b92e9f9d14eafdec8 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 6 Apr 2020 12:05:57 -0400 Subject: [PATCH 125/282] Ran `pre-commit autoupdate` to get the latest version of the `pre-commit-terraform` hook since the PR I submitted, https://github.com/antonbabenko/pre-commit-terraform/pull/100, was approved. This will fix issues with `skeleton-tf-module` related to multiple directories with Terraform code. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 42b824f..129aa39 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -76,7 +76,7 @@ repos: - id: ansible-lint # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git - rev: v1.27.0 + rev: v1.29.0 hooks: - id: terraform_fmt - id: terraform_validate @@ -85,7 +85,7 @@ repos: hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier - rev: 2.0.2 + rev: 2.0.4 hooks: - id: prettier - repo: https://github.com/pre-commit/mirrors-mypy From b01a0ee7bd1e0854b4c4c6bb999316c21221c97f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 6 Apr 2020 13:47:18 -0400 Subject: [PATCH 126/282] Explicitly install pre-commit hooks as its own step so it's clearer when the failure is with setup rather than hooks running. --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 298e93c..6026d47 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,5 +42,7 @@ jobs: run: | python -m pip install --upgrade pip pip install --upgrade --requirement requirements-test.txt + - name: Install pre-commit hooks + run: pre-commit install-hooks - name: Run pre-commit on all files run: pre-commit run --all-files From 1f3d440a1af6283621b5584da9b56e2747d40a0b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 6 Apr 2020 19:02:31 -0400 Subject: [PATCH 127/282] Rename action to accurately reflect that we are only linting, not building. Adjust description for pre-commit hook setup to better convey what is being done. --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6026d47..9c6b03a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,7 +11,7 @@ env: PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit jobs: - build: + lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -42,7 +42,7 @@ jobs: run: | python -m pip install --upgrade pip pip install --upgrade --requirement requirements-test.txt - - name: Install pre-commit hooks + - name: Set up pre-commit hook environments run: pre-commit install-hooks - name: Run pre-commit on all files run: pre-commit run --all-files From 647b351997ab7cafd391bdabda7b1b5e68214b8a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 6 Apr 2020 19:04:21 -0400 Subject: [PATCH 128/282] Sort .gitignore entries. --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 95b74cd..e00826d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ .mypy_cache -__pycache__ .python-version +__pycache__ From a988a2d133de1f55ccc85e2496681346688dcef6 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 7 Apr 2020 13:54:01 -0400 Subject: [PATCH 129/282] Remove duplicate '.mypy_cache' entry in .gitignore --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index ad2861f..fdbe25e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ *.egg-info .coverage .mypy_cache -.mypy_cache .pytest_cache .python-version __pycache__ From c8b4bcf126989abf35e4ce226263ef7da139c959 Mon Sep 17 00:00:00 2001 From: Hillary Date: Tue, 14 Apr 2020 12:43:18 -0400 Subject: [PATCH 130/282] Update CODEOWNERS --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 32918e8..a3619be 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,4 +4,4 @@ # the repo. Unless a later match takes precedence, # these owners will be requested for review when someone # opens a pull request. -* @dav3r @felddy @jsf9k @mcdonnnj @cisagov/team-ois +* @dav3r @felddy @hillaryj @jsf9k @mcdonnnj @cisagov/team-ois From 8c99c5294a798bca98a68212507fce924505a289 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 Apr 2020 23:31:59 -0400 Subject: [PATCH 131/282] Allow events from apb to rebuild this repository weekly. --- .github/workflows/build.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9c6b03a..5fe2f8b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,10 +1,11 @@ --- name: build -on: [ - push, - pull_request -] +on: + push: + pull_request: + repository_dispatch: + types: [apb] env: PIP_CACHE_DIR: ~/.cache/pip From 7d7a6774571c4727b4858b1117c31f97160131da Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 8 May 2020 17:49:17 -0400 Subject: [PATCH 132/282] Updated pre-commit configuration with 'pre-commit autoupdate'. --- .pre-commit-config.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 129aa39..de0152b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,13 +27,13 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.22.0 + rev: v0.23.0 hooks: - id: markdownlint args: - --config=.mdl_config.json - repo: https://github.com/adrienverge/yamllint - rev: v1.21.0 + rev: v1.23.0 hooks: - id: yamllint - repo: https://github.com/detailyang/pre-commit-shell @@ -41,13 +41,13 @@ repos: hooks: - id: shell-lint - repo: https://gitlab.com/pycqa/flake8 - rev: 3.7.9 + rev: 3.8.0a2 hooks: - id: flake8 additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v2.1.0 + rev: v2.4.1 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit @@ -61,7 +61,7 @@ repos: hooks: - id: black - repo: https://github.com/asottile/seed-isort-config - rev: v2.1.0 + rev: v2.1.1 hooks: - id: seed-isort-config - repo: https://github.com/pre-commit/mirrors-isort @@ -71,12 +71,12 @@ repos: hooks: - id: isort - repo: https://github.com/ansible/ansible-lint.git - rev: v4.2.0 + rev: v4.3.0a0 hooks: - id: ansible-lint # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git - rev: v1.29.0 + rev: v1.30.0 hooks: - id: terraform_fmt - id: terraform_validate @@ -85,7 +85,7 @@ repos: hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier - rev: 2.0.4 + rev: 2.0.5 hooks: - id: prettier - repo: https://github.com/pre-commit/mirrors-mypy From b724c79e29f2d84e22438cd15803c5db08049472 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 11 May 2020 21:44:53 -0400 Subject: [PATCH 133/282] Add Lineage configuration. --- .github/lineage.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .github/lineage.yml diff --git a/.github/lineage.yml b/.github/lineage.yml new file mode 100644 index 0000000..8dfc20b --- /dev/null +++ b/.github/lineage.yml @@ -0,0 +1,6 @@ +--- +version: "1" + +lineage: + skeleton: + remote-url: https://github.com/cisagov/skeleton-generic.git From ce6658abc69d1444e51f45c20441c98e6b54c086 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 13 May 2020 15:25:31 -0400 Subject: [PATCH 134/282] Update isort pre-commit hook source repository. The mirrors-isort respository has been deprecated per this Pull Request: https://github.com/pre-commit/mirrors-isort/pull/13 Since isort includes pre-commit configuration in current versions we will switch to using it directly. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index de0152b..5be35e4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -64,10 +64,10 @@ repos: rev: v2.1.1 hooks: - id: seed-isort-config - - repo: https://github.com/pre-commit/mirrors-isort + - repo: https://github.com/timothycrosley/isort # pick the isort version you'd like to use from # https://github.com/pre-commit/mirrors-isort/releases - rev: v4.3.21 + rev: 4.3.21 hooks: - id: isort - repo: https://github.com/ansible/ansible-lint.git From daaebc1cc13782e8fb39be6815aa53bf3efd44f3 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 13 May 2020 15:30:04 -0400 Subject: [PATCH 135/282] Ran pre-commit autoupdate. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5be35e4..a2e4f76 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,7 +41,7 @@ repos: hooks: - id: shell-lint - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.0a2 + rev: 3.8.1 hooks: - id: flake8 additional_dependencies: From 2399f24622b1d6be92c494653d9308e9694c5496 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 13 May 2020 15:35:36 -0400 Subject: [PATCH 136/282] Remove legacy comment from the isort hook declaration. This comment references the mirrors-isort repository that we are no longer using. --- .pre-commit-config.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a2e4f76..bf8ae6e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -65,8 +65,6 @@ repos: hooks: - id: seed-isort-config - repo: https://github.com/timothycrosley/isort - # pick the isort version you'd like to use from - # https://github.com/pre-commit/mirrors-isort/releases rev: 4.3.21 hooks: - id: isort From fe4d5eb72a8f6ce9e23daae98d7a0c87675a9da4 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 13 May 2020 16:07:18 -0400 Subject: [PATCH 137/282] Disable the terraform_validate hook. We have seen a number of issues related to this hook ever since it was re-enabled. It will need to remain disabled until at least the 0.13 Terraform release, and can only be re-enabled if all issues we have seen have been resolved in how `terraform validate` operates. --- .pre-commit-config.yaml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bf8ae6e..74af27c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -77,7 +77,19 @@ repos: rev: v1.30.0 hooks: - id: terraform_fmt - - id: terraform_validate + # There are ongoing issues with how this command works. This issue + # documents the core issue: + # https://github.com/hashicorp/terraform/issues/21408 + # We have seen issues primarily with proxy providers and Terraform code + # that uses remote state. The PR + # https://github.com/hashicorp/terraform/pull/24887 + # has been approved and is part of the 0.13 release to resolve the issue + # with remote states. + # The PR + # https://github.com/hashicorp/terraform/pull/24896 + # is a proprosed fix to deal with `terraform validate` with proxy + # providers (among other configurations). + # - id: terraform_validate - repo: https://github.com/IamTheFij/docker-pre-commit rev: v1.0.1 hooks: From f831ef827cae16fd288f2e7a16fedc53c6e06873 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 13 May 2020 21:34:58 -0400 Subject: [PATCH 138/282] Add final statement to descriptionf or why terraform_validate is disabled. Review noticed that there lacked a determination for what we were doing about the problem with the terraform_validate hook. I described the problems but failed to mention what our path forward would be. This commit rectifies that oversight. --- .pre-commit-config.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 74af27c..b7c5518 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -89,6 +89,9 @@ repos: # https://github.com/hashicorp/terraform/pull/24896 # is a proprosed fix to deal with `terraform validate` with proxy # providers (among other configurations). + # We have decided to disable the terraform_validate hook until the issues + # above have been resolved, which we hope will be with the release of + # Terraform 0.13. # - id: terraform_validate - repo: https://github.com/IamTheFij/docker-pre-commit rev: v1.0.1 From 87ea53022c39f3adead2688fbae5698147b279d8 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 9 Jun 2020 15:10:11 -0400 Subject: [PATCH 139/282] Add setuptools and wheel as pip dependencies setuptools usually comes along with pip, but wheel does not. Using wheel where possible to build python extensions is more modern and more security conscious than using setup.py. --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0a8547b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +setuptools +wheel From 7f8d430a309b22a3410949e64ccd60ad5a1dc79f Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 9 Jun 2020 15:19:05 -0400 Subject: [PATCH 140/282] Pull in requirements.txt from requirements-test.txt --- requirements-test.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-test.txt b/requirements-test.txt index 416634f..66f74db 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1 +1,2 @@ +--requirement requirements.txt pre-commit From 370f2a2d8b7375c6e529565aa49a41d6499c028e Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 15 Jun 2020 17:35:25 -0400 Subject: [PATCH 141/282] Remove requirements that already exist in setup.py to align with a Single Source of Truth. --- requirements-test.txt | 1 - requirements.txt | 1 - 2 files changed, 2 deletions(-) diff --git a/requirements-test.txt b/requirements-test.txt index 47e4ffd..0a6d7d6 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,3 +1,2 @@ --requirement requirements.txt -e .[test] -pre-commit diff --git a/requirements.txt b/requirements.txt index e9d0dcc..d08662f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ -e . -setuptools wheel From d2d05000b14b1d8bffc83040d268f30faf03a9d4 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 18 Jun 2020 21:46:17 -0400 Subject: [PATCH 142/282] Switch test and build workflows to use a matrix of Python versions. We claim to support Python versions 3.6, 3.7, and 3.8 in setup.py, but we only test and build on Python 3.8. If we claim support for a version then I believe we should be testing and building on that version to ensure "advertised" compatibility. --- .github/workflows/build.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7237b5a..fe5337e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -49,11 +49,14 @@ jobs: run: pre-commit run --all-files test: runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.6, 3.7, 3.8] steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: ${{ matrix.python_version }} - name: Store installed Python version run: | echo "::set-env name=PY_VERSION::"\ @@ -84,11 +87,14 @@ jobs: build: runs-on: ubuntu-latest needs: [lint, test] + strategy: + matrix: + python-version: [3.6, 3.7, 3.8] steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: ${{ matrix.python-version }} - name: Store installed Python version run: | echo "::set-env name=PY_VERSION::"\ @@ -112,5 +118,5 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v1 with: - name: dist + name: dist-${{ matrix.python-version }} path: dist From 6f6f5972fac972fc150286d571eeb5aabb44a2fc Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 18 Jun 2020 21:56:58 -0400 Subject: [PATCH 143/282] Fix typo and use matrix.python-version instead of env.PY_VERSION. In workflows using the Python version matrix we should just use the version for that run instead of extracting the Python version from the environment. --- .github/workflows/build.yml | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fe5337e..8bf777b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -56,19 +56,15 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v1 with: - python-version: ${{ matrix.python_version }} - - name: Store installed Python version - run: | - echo "::set-env name=PY_VERSION::"\ - "$(python -c "import platform;print(platform.python_version())")" + python-version: ${{ matrix.python-version }} - name: Cache pip test requirements uses: actions/cache@v1 with: path: ${{ env.PIP_CACHE_DIR }} - key: "${{ runner.os }}-pip-test-py${{ env.PY_VERSION }}-\ + key: "${{ runner.os }}-pip-test-py${{ matrix.python-version }}-\ ${{ hashFiles('**/requirements-test.txt') }}" restore-keys: | - ${{ runner.os }}-pip-test-py${{ env.PY_VERSION }}- + ${{ runner.os }}-pip-test-py${{ matrix.python-version }}- ${{ runner.os }}-pip-test- ${{ runner.os }}-pip- - name: Install dependencies @@ -95,18 +91,14 @@ jobs: - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - - name: Store installed Python version - run: | - echo "::set-env name=PY_VERSION::"\ - "$(python -c "import platform;print(platform.python_version())")" - name: Cache pip build requirements uses: actions/cache@v1 with: path: ${{ env.PIP_CACHE_DIR }} - key: "${{ runner.os }}-pip-build-py${{ env.PY_VERSION }}-\ + key: "${{ runner.os }}-pip-build-py${{ matrix.python-version }}-\ ${{ hashFiles('**/requirements.txt') }}" restore-keys: | - ${{ runner.os }}-pip-build-py${{ env.PY_VERSION }}- + ${{ runner.os }}-pip-build-py${{ matrix.python-version }}- ${{ runner.os }}-pip-build- ${{ runner.os }}-pip- - name: Install dependencies From af06915b05674695cb5d9bd8418c6b37f77ec3bc Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 14:57:04 -0400 Subject: [PATCH 144/282] Add tool and docs to automate development environment setup. --- CONTRIBUTING.md | 11 +++ setup-env | 175 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 186 insertions(+) create mode 100755 setup-env diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dacaaad..2bd2451 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,6 +46,17 @@ There are a few ways to do this, but we prefer to use create and manage a Python virtual environment specific to this project. +If you already have `pyenv` and `pyenv-virtualenv` configured you can +take advantage of the `setup-env` tool in this repo to automate the +entire environment configuration process. + +```bash +./setup-env +``` + +Otherwise, follow the steps below to manually configure your +environment. + #### Installing and using `pyenv` and `pyenv-virtualenv` #### On the Mac, installation is as simple as `brew install pyenv diff --git a/setup-env b/setup-env new file mode 100755 index 0000000..65c5b6f --- /dev/null +++ b/setup-env @@ -0,0 +1,175 @@ +#!/usr/bin/env bash + +set -o nounset +set -o errexit +set -o pipefail + +USAGE=$(cat << 'END_OF_LINE' +This script is used to configure a developement environment for this repo. + +It does the following: + - Verifies pyenv and pyenv-virtualenv are installed. + - Creates a Python virtual environment. + - Configures the activation of the virtual enviroment for the repo directory. + - Installs the requirements required for development. + - Installs git pre-commit hooks. + - Configures git upstream remote "lineage" repositories. + +usage: setup-env [--force] [--help] [virt_env_name] + +END_OF_LINE +) + +# Flag to force deletion and creation of virtual environment +FORCE=0 + +# Positional parameters +PARAMS="" + +# Parse command line arguments +while (( "$#" )); do + case "$1" in + -f|--force) + FORCE=1 + shift + ;; + -h|--help) + echo "${USAGE}" + exit 0 + ;; + -*) # unsupported flags + echo "Error: Unsupported flag $1" >&2 + exit 1 + ;; + *) # preserve positional arguments + PARAMS="$PARAMS $1" + shift + ;; + esac +done + +# set positional arguments in their proper place +eval set -- "$PARAMS" + +# Check to see if pyenv is installed +if [ -z "$(which pyenv)" ] || [ -z "$(which pyenv-virtualenv)" ]; then + echo "pyenv and pyenv-virtualenv are required." + if [[ "$OSTYPE" == "darwin"* ]]; then + cat << 'END_OF_LINE' + + On the Mac, installation is as simple as "brew install pyenv + pyenv-virtualenv" and adding this to your profile: + + eval "$(pyenv init -)" + eval "$(pyenv virtualenv-init -)" + +END_OF_LINE + + fi + cat << 'END_OF_LINE' + For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you don't want + to use "brew") you can use https://github.com/pyenv/pyenv-installer to install + the necessary tools. Before running this ensure that you have installed the + prerequisites for your platform according to the pyenv wiki page, + https://github.com/pyenv/pyenv/wiki/common-build-problems. + + On WSL you should treat your platform as whatever Linux distribution you've + chosen to install. + + Once you have installed "pyenv" you will need to add the following lines to + your ".bashrc": + + export PATH="$PATH:$HOME/.pyenv/bin" + eval "$(pyenv init -)" + eval "$(pyenv virtualenv-init -)" +END_OF_LINE + exit 1 +fi + +set +o nounset +# Determine the virtual environment name +if [ "$1" ]; then + # Use the user-provided environment name + env_name=$1 +else + # Set the environment name to the last part of the working directory. + env_name=${PWD##*/} +fi +set -o nounset + +# Remove any lingering local configuration. +if [ $FORCE -ne 0 ]; then + rm -f .python-version + pyenv virtualenv-delete --force "${env_name}" || true +elif [[ -f .python-version ]]; then + cat << 'END_OF_LINE' + An existing .python-version file was found. Either remove this file yourself + or re-run with --force option to have it deleted along with the associated + virtual environment. + + rm .python-version + +END_OF_LINE + exit 1 +fi + +# Create a new virutal environment for this project +if ! pyenv virtualenv "${env_name}"; then + cat << END_OF_LINE + An existing virtual environment named $env_name was found. Either delete this + environment yourself or re-run with --force option to have it deleted. + + pyenv virtualenv-delete ${env_name} + +END_OF_LINE + exit 1 +fi + +# Activate the new virtual environment +pyenv local "${env_name}" + +# Upgrade pip and friends +python -m pip install --upgrade pip setuptools wheel + +# Find a requirements file (if possible) and install +for req_file in "requirements-dev.txt" "requirements-test.txt" "requirements.txt"; do + if [[ -f $req_file ]] + then + pip install -r $req_file + break + fi +done + +# Install git pre-commit hooks +pre-commit install + +# Setup git remotes from lineage configuration +# This could fail if the remotes are already setup, but that is ok. +set +o errexit + +eval "$(python3 << 'END_OF_LINE' +from pathlib import Path +import yaml +import sys + +LINEAGE_CONFIG = Path(".github/lineage.yml") + +if not LINEAGE_CONFIG.exists(): + print('No lineage configuration found.', file=sys.stderr) + sys.exit(0) + +with LINEAGE_CONFIG.open("r") as f: + lineage = yaml.safe_load(stream=f) + +if lineage["version"] == "1": + for parent_name, v in lineage["lineage"].items(): + remote_url = v["remote-url"] + print(f"git remote add {parent_name} {remote_url};") + print(f"git remote set-url --push {parent_name} no_push;") +else: + print(f'Unsupported lineage version: {lineage["version"]}', file=sys.stderr) +END_OF_LINE +)" + +# Qapla +echo "Success!" From c50094a8dee2573462c355e82e01eba13fb1fb14 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 15:15:18 -0400 Subject: [PATCH 145/282] Fix typo. --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 65c5b6f..5015beb 100755 --- a/setup-env +++ b/setup-env @@ -113,7 +113,7 @@ END_OF_LINE exit 1 fi -# Create a new virutal environment for this project +# Create a new virtual environment for this project if ! pyenv virtualenv "${env_name}"; then cat << END_OF_LINE An existing virtual environment named $env_name was found. Either delete this From 3c2781c8cc3d87d4e1433100e243679e899d7fe1 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 15:25:59 -0400 Subject: [PATCH 146/282] Correct syntax on fence. Closes CVE-2020-3.14159 --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2bd2451..8635fea 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,7 +50,7 @@ If you already have `pyenv` and `pyenv-virtualenv` configured you can take advantage of the `setup-env` tool in this repo to automate the entire environment configuration process. -```bash +```console ./setup-env ``` From 23ef29c8a301a6851664ae0ccd8633363f95648f Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 16:30:46 -0400 Subject: [PATCH 147/282] Clarify comment with text from `pyenv local --help` --- setup-env | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 5015beb..5f404b3 100755 --- a/setup-env +++ b/setup-env @@ -125,7 +125,8 @@ END_OF_LINE exit 1 fi -# Activate the new virtual environment +# Set the local application-specific Python version(s) by writing the +# version name to a file named `.python-version'. pyenv local "${env_name}" # Upgrade pip and friends From 5d1d134c88aa203d69d4baf88e9b1e5d4b7a2eab Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 16:34:53 -0400 Subject: [PATCH 148/282] Change to python3 to standardize how we call python. --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 5f404b3..a767b3d 100755 --- a/setup-env +++ b/setup-env @@ -130,7 +130,7 @@ fi pyenv local "${env_name}" # Upgrade pip and friends -python -m pip install --upgrade pip setuptools wheel +python3 -m pip install --upgrade pip setuptools wheel # Find a requirements file (if possible) and install for req_file in "requirements-dev.txt" "requirements-test.txt" "requirements.txt"; do From 8a105ca04010ca92cd066a525b840fe1c6de66f5 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 16:50:42 -0400 Subject: [PATCH 149/282] Respect customs. --- setup-env | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup-env b/setup-env index a767b3d..713687a 100755 --- a/setup-env +++ b/setup-env @@ -134,9 +134,8 @@ python3 -m pip install --upgrade pip setuptools wheel # Find a requirements file (if possible) and install for req_file in "requirements-dev.txt" "requirements-test.txt" "requirements.txt"; do - if [[ -f $req_file ]] - then - pip install -r $req_file + if [[ -f $req_file ]]; then + pip install --requirement $req_file break fi done From ae21af108405bde6ea772d804f71ecf3cd677895 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 16:55:01 -0400 Subject: [PATCH 150/282] Improve usage message. --- setup-env | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/setup-env b/setup-env index 713687a..7bc99f9 100755 --- a/setup-env +++ b/setup-env @@ -5,17 +5,25 @@ set -o errexit set -o pipefail USAGE=$(cat << 'END_OF_LINE' -This script is used to configure a developement environment for this repo. +Configure a developement environment for this repository. It does the following: - Verifies pyenv and pyenv-virtualenv are installed. - Creates a Python virtual environment. - Configures the activation of the virtual enviroment for the repo directory. - - Installs the requirements required for development. + - Installs the requirements needed for development. - Installs git pre-commit hooks. - Configures git upstream remote "lineage" repositories. -usage: setup-env [--force] [--help] [virt_env_name] +Usage: + setup-env [--force] [virt_env_name] + setup-env (-h | --help) + +Options: + -f --force Delete virtual enviroment if it already exists. + -h --help Show this message. + -i --install-hooks Install hook environments for all environments in the + pre-commit config file. END_OF_LINE ) From c5e7edf46b736965216b258ac562cbf6ad6e1218 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 16:55:34 -0400 Subject: [PATCH 151/282] Add flag to install pre-commit hooks now. --- setup-env | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/setup-env b/setup-env index 7bc99f9..5b6dacd 100755 --- a/setup-env +++ b/setup-env @@ -37,6 +37,10 @@ PARAMS="" # Parse command line arguments while (( "$#" )); do case "$1" in + -i|--install-hooks) + INSTALL_HOOKS=1 + shift + ;; -f|--force) FORCE=1 shift @@ -148,8 +152,8 @@ for req_file in "requirements-dev.txt" "requirements-test.txt" "requirements.txt fi done -# Install git pre-commit hooks -pre-commit install +# Install git pre-commit hooks now or later. +pre-commit install ${INSTALL_HOOKS:+"--install-hooks"} # Setup git remotes from lineage configuration # This could fail if the remotes are already setup, but that is ok. From 482a370679570e55233c13ef6df644ee3437694e Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 17:02:03 -0400 Subject: [PATCH 152/282] Switch to a POSIX compliant command... command. This is the way. I learned something today. https://unix.stackexchange.com/questions/85249/why-not-use-which-what-to-use-then --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 5b6dacd..9d1ed6a 100755 --- a/setup-env +++ b/setup-env @@ -64,7 +64,7 @@ done eval set -- "$PARAMS" # Check to see if pyenv is installed -if [ -z "$(which pyenv)" ] || [ -z "$(which pyenv-virtualenv)" ]; then +if [ -z "$(command -v pyenv)" ] || [ -z "$(command -v pyenv-virtualenv)" ]; then echo "pyenv and pyenv-virtualenv are required." if [[ "$OSTYPE" == "darwin"* ]]; then cat << 'END_OF_LINE' From 466ef0cce560fc18b4d657caef943f9c07626fc8 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 17:05:19 -0400 Subject: [PATCH 153/282] Sort command line flag case statement. --- setup-env | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup-env b/setup-env index 9d1ed6a..0b44a24 100755 --- a/setup-env +++ b/setup-env @@ -37,10 +37,6 @@ PARAMS="" # Parse command line arguments while (( "$#" )); do case "$1" in - -i|--install-hooks) - INSTALL_HOOKS=1 - shift - ;; -f|--force) FORCE=1 shift @@ -49,6 +45,10 @@ while (( "$#" )); do echo "${USAGE}" exit 0 ;; + -i|--install-hooks) + INSTALL_HOOKS=1 + shift + ;; -*) # unsupported flags echo "Error: Unsupported flag $1" >&2 exit 1 From 996ae2677a33e1631c8ac80a6e9c682be22e6ac9 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 17:13:51 -0400 Subject: [PATCH 154/282] Add recommendation to install brew, instead of assuming it is available. --- CONTRIBUTING.md | 5 +++-- setup-env | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8635fea..6434d30 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -59,8 +59,9 @@ environment. #### Installing and using `pyenv` and `pyenv-virtualenv` #### -On the Mac, installation is as simple as `brew install pyenv -pyenv-virtualenv` and adding this to your profile: +On the Mac, we recommend installing [brew](https://brew.sh/). Then +installation is as simple as `brew install pyenv pyenv-virtualenv` and +adding this to your profile: ```bash eval "$(pyenv init -)" diff --git a/setup-env b/setup-env index 0b44a24..46620e0 100755 --- a/setup-env +++ b/setup-env @@ -69,8 +69,9 @@ if [ -z "$(command -v pyenv)" ] || [ -z "$(command -v pyenv-virtualenv)" ]; then if [[ "$OSTYPE" == "darwin"* ]]; then cat << 'END_OF_LINE' - On the Mac, installation is as simple as "brew install pyenv - pyenv-virtualenv" and adding this to your profile: + On the Mac, we recommend installing brew, https://brew.sh/. Then installation + is as simple as `brew install pyenv pyenv-virtualenv` and adding this to your + profile: eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" From 31980ed4a8624baa682735d0ecfef611582e55a1 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 17:18:19 -0400 Subject: [PATCH 155/282] Clean up usage syntax since we now have multiple options. --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 46620e0..57abe56 100755 --- a/setup-env +++ b/setup-env @@ -16,7 +16,7 @@ It does the following: - Configures git upstream remote "lineage" repositories. Usage: - setup-env [--force] [virt_env_name] + setup-env [options] [virt_env_name] setup-env (-h | --help) Options: From db362b912512418280e8b74891de812016e4fb9f Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 17:37:34 -0400 Subject: [PATCH 156/282] Normalize quotes. --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 57abe56..4d822c4 100755 --- a/setup-env +++ b/setup-env @@ -168,7 +168,7 @@ import sys LINEAGE_CONFIG = Path(".github/lineage.yml") if not LINEAGE_CONFIG.exists(): - print('No lineage configuration found.', file=sys.stderr) + print("No lineage configuration found.", file=sys.stderr) sys.exit(0) with LINEAGE_CONFIG.open("r") as f: From 91279d3ac2a17383acbea2f82a193b3fefdaa895 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 10 Jul 2020 21:28:58 -0400 Subject: [PATCH 157/282] Perform a pre-commit autoupdate for 2020-07 Notable version changes: - docker-pre-commit v2.0.0 This release includes the addition of two new hooks: hadolint and hadolint-system. These allow for linting Dockerfiles with the hadolint Docker image or a system installation of hadolint respectively. - isort 5.0.7 This release takes us from isort 4 to isort 5. There are a multitude of changes with information available on the isort 5 release page at https://timothycrosley.github.io/isort/docs/major_releases/introducing_isort_5/ The most notable feature for us is the addition of built-in profiles for isort to run under. These profiles provide settings for conforming to various popular styles, and black is one of the profiles. --- .pre-commit-config.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b7c5518..52b6473 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.5.0 + rev: v3.1.0 hooks: - id: check-executables-have-shebangs - id: check-json @@ -27,7 +27,7 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.23.0 + rev: v0.23.2 hooks: - id: markdownlint args: @@ -41,13 +41,13 @@ repos: hooks: - id: shell-lint - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.1 + rev: 3.8.3 hooks: - id: flake8 additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v2.4.1 + rev: v2.7.0 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit @@ -61,20 +61,20 @@ repos: hooks: - id: black - repo: https://github.com/asottile/seed-isort-config - rev: v2.1.1 + rev: v2.2.0 hooks: - id: seed-isort-config - repo: https://github.com/timothycrosley/isort - rev: 4.3.21 + rev: 5.0.7 hooks: - id: isort - repo: https://github.com/ansible/ansible-lint.git - rev: v4.3.0a0 + rev: v4.3.0a3 hooks: - id: ansible-lint # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git - rev: v1.30.0 + rev: v1.31.0 hooks: - id: terraform_fmt # There are ongoing issues with how this command works. This issue @@ -94,7 +94,7 @@ repos: # Terraform 0.13. # - id: terraform_validate - repo: https://github.com/IamTheFij/docker-pre-commit - rev: v1.0.1 + rev: v2.0.0 hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier @@ -102,6 +102,6 @@ repos: hooks: - id: prettier - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.770 + rev: v0.782 hooks: - id: mypy From 907b89a20b17b5f8ffadbdc7ee71ed6cc31fd63b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 10 Jul 2020 21:34:22 -0400 Subject: [PATCH 158/282] Add the black profile to the isort configuration --- .isort.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.isort.cfg b/.isort.cfg index 5216e47..2fd6b08 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -10,3 +10,6 @@ import_heading_firstparty=cisagov Libraries known_third_party= # These must be manually set to correctly separate them from third party libraries known_first_party= + +# Run isort under the black profile to align with our other Python linting +profile=black From 2d88250093549fe08df66583cf5877dc309d43fd Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 13 Jul 2020 10:04:06 -0400 Subject: [PATCH 159/282] Run isort against the tests/ directory --- tests/test_example.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_example.py b/tests/test_example.py index c3334d7..689385d 100644 --- a/tests/test_example.py +++ b/tests/test_example.py @@ -90,6 +90,7 @@ def test_slow_division(): This test will only be run if --runslow is passed to pytest. Look in conftest.py to see how this is implemented. """ + # Standard Python Libraries import time result = example.example_div(256, 16) From d6fe79dce37d8ecab907d7ba49fd580d4191819b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 30 Jul 2020 02:19:02 -0400 Subject: [PATCH 160/282] Update actions/cache to v2 Switch the workflow to use cachev2. This puts us up to date and allows us to consolidate into one cache item. --- .github/workflows/build.yml | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5fe2f8b..f6f4474 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,22 +23,18 @@ jobs: run: | echo "::set-env name=PY_VERSION::"\ "$(python -c "import platform;print(platform.python_version())")" - - name: Cache pip test requirements - uses: actions/cache@v1 + - name: Cache linting environments + uses: actions/cache@v2 with: - path: ${{ env.PIP_CACHE_DIR }} - key: "${{ runner.os }}-pip-test-py${{ env.PY_VERSION }}-\ - ${{ hashFiles('**/requirements-test.txt') }}" - restore-keys: | - ${{ runner.os }}-pip-test-py${{ env.PY_VERSION }}- - ${{ runner.os }}-pip-test- - ${{ runner.os }}-pip- - - name: Cache pre-commit hooks - uses: actions/cache@v1 - with: - path: ${{ env.PRE_COMMIT_CACHE_DIR }} - key: "${{ runner.os }}-pre-commit-py${{ env.PY_VERSION }}-\ + path: | + ${{ env.PIP_CACHE_DIR }} + ${{ env.PRE_COMMIT_CACHE_DIR }} + key: "${{ runner.os }}-lint-py${{ env.PY_VERSION }}-\ + ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" + restore-keys: | + ${{ runner.os }}-lint-py${{ env.PY_VERSION }}- + ${{ runner.os }}-lint- - name: Install dependencies run: | python -m pip install --upgrade pip From 6ab69f4f5158c3b2681f1b78a8fd71a57f0eb4d0 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 30 Jul 2020 02:21:04 -0400 Subject: [PATCH 161/282] Bump actions/setup-python to v2 Keep it up-to-date even though we are not currently using features added in v2. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f6f4474..2f58ed6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v2 with: python-version: 3.8 - name: Store installed Python version From 969efdd0bbc8cfad942ab0b731e24dcd43b8864f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 30 Jul 2020 11:08:22 -0400 Subject: [PATCH 162/282] Adjust cache key name Make the job name the first part of the cache key since it is the only part that isn't variable. --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2f58ed6..36a908a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,12 +29,12 @@ jobs: path: | ${{ env.PIP_CACHE_DIR }} ${{ env.PRE_COMMIT_CACHE_DIR }} - key: "${{ runner.os }}-lint-py${{ env.PY_VERSION }}-\ + key: "lint-${{ runner.os }}-py${{ env.PY_VERSION }}-\ ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" restore-keys: | - ${{ runner.os }}-lint-py${{ env.PY_VERSION }}- - ${{ runner.os }}-lint- + lint-${{ runner.os }}-py${{ env.PY_VERSION }}- + lint-${{ runner.os }}- - name: Install dependencies run: | python -m pip install --upgrade pip From 3004218129e6cccfc1d0e1a5aae071f0eebba7fc Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 30 Jul 2020 14:32:53 -0400 Subject: [PATCH 163/282] Update test and build jobs to mirror changes to lint All jobs should perform identical work in a consistent manner. --- .github/workflows/build.yml | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b057676..09b855a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -50,19 +50,18 @@ jobs: python-version: [3.6, 3.7, 3.8] steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Cache pip test requirements - uses: actions/cache@v1 + - name: Cache testing environments + uses: actions/cache@v2 with: path: ${{ env.PIP_CACHE_DIR }} - key: "${{ runner.os }}-pip-test-py${{ matrix.python-version }}-\ + key: "test-${{ runner.os }}-py${{ matrix.python-version }}-\ ${{ hashFiles('**/requirements-test.txt') }}" restore-keys: | - ${{ runner.os }}-pip-test-py${{ matrix.python-version }}- - ${{ runner.os }}-pip-test- - ${{ runner.os }}-pip- + test-${{ runner.os }}-py${{ matrix.python-version }}- + test-${{ runner.os }}- - name: Install dependencies run: | python -m pip install --upgrade pip @@ -84,19 +83,18 @@ jobs: python-version: [3.6, 3.7, 3.8] steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Cache pip build requirements - uses: actions/cache@v1 + - name: Cache building environments + uses: actions/cache@v2 with: path: ${{ env.PIP_CACHE_DIR }} - key: "${{ runner.os }}-pip-build-py${{ matrix.python-version }}-\ + key: "build-${{ runner.os }}-py${{ matrix.python-version }}-\ ${{ hashFiles('**/requirements.txt') }}" restore-keys: | - ${{ runner.os }}-pip-build-py${{ matrix.python-version }}- - ${{ runner.os }}-pip-build- - ${{ runner.os }}-pip- + build-${{ runner.os }}-py${{ matrix.python-version }}- + build-${{ runner.os }}- - name: Install dependencies run: | python -m pip install --upgrade pip wheel From 332e5a7669c39eff0feef36d054acb5766069fd7 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 3 Sep 2020 16:27:01 -0400 Subject: [PATCH 164/282] Run pre-commit autoupdate --- .pre-commit-config.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 52b6473..2689bb0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.1.0 + rev: v3.2.0 hooks: - id: check-executables-have-shebangs - id: check-json @@ -33,7 +33,7 @@ repos: args: - --config=.mdl_config.json - repo: https://github.com/adrienverge/yamllint - rev: v1.23.0 + rev: v1.24.2 hooks: - id: yamllint - repo: https://github.com/detailyang/pre-commit-shell @@ -47,7 +47,7 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v2.7.0 + rev: v2.7.2 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit @@ -57,7 +57,7 @@ repos: args: - --config=.bandit.yml - repo: https://github.com/python/black - rev: 19.10b0 + rev: 20.8b1 hooks: - id: black - repo: https://github.com/asottile/seed-isort-config @@ -65,16 +65,16 @@ repos: hooks: - id: seed-isort-config - repo: https://github.com/timothycrosley/isort - rev: 5.0.7 + rev: 5.5.0 hooks: - id: isort - repo: https://github.com/ansible/ansible-lint.git - rev: v4.3.0a3 + rev: v4.3.4 hooks: - id: ansible-lint # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git - rev: v1.31.0 + rev: v1.37.0 hooks: - id: terraform_fmt # There are ongoing issues with how this command works. This issue @@ -98,7 +98,7 @@ repos: hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier - rev: 2.0.5 + rev: 2.1.1 hooks: - id: prettier - repo: https://github.com/pre-commit/mirrors-mypy From 3e9441c837ce7d34a4043a8348f1bbcccaf2608b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 3 Sep 2020 16:29:38 -0400 Subject: [PATCH 165/282] Update CODEOWNERS to resolve #50 --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a3619be..7735a52 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,4 +4,4 @@ # the repo. Unless a later match takes precedence, # these owners will be requested for review when someone # opens a pull request. -* @dav3r @felddy @hillaryj @jsf9k @mcdonnnj @cisagov/team-ois +* @dav3r @felddy @hillaryj @jsf9k @mcdonnnj From ae4aa123c81c0915bb1ce3da2ee19b41a98c9846 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 3 Sep 2020 16:31:23 -0400 Subject: [PATCH 166/282] Update lint job to resolve #49 --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 36a908a..7a92841 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,6 +31,7 @@ jobs: ${{ env.PRE_COMMIT_CACHE_DIR }} key: "lint-${{ runner.os }}-py${{ env.PY_VERSION }}-\ ${{ hashFiles('**/requirements-test.txt') }}-\ + ${{ hashFiles('**/requirements.txt') }}-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" restore-keys: | lint-${{ runner.os }}-py${{ env.PY_VERSION }}- From d8f14e845b4054e8a45fc1f9a38c8537bcf30bfb Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 3 Sep 2020 16:32:30 -0400 Subject: [PATCH 167/282] Sort .gitignore while I'm here --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e00826d..4b15d77 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ +__pycache__ .mypy_cache .python-version -__pycache__ From 26b66e4468296a3b6765d63c0d64e5ee205a82b2 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 4 Sep 2020 10:34:46 -0400 Subject: [PATCH 168/282] Update build workflow and change pip option Add "requirements.txt" to the actions/cache key for the test job to match what was done for the lint job. Expand the option to pip in requirements.txt to its full, more descriptive form. This matches efforts to do this elsewhere in our codebases. --- .github/workflows/build.yml | 3 ++- requirements.txt | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ab554d4..6ad7ec9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -59,7 +59,8 @@ jobs: with: path: ${{ env.PIP_CACHE_DIR }} key: "test-${{ runner.os }}-py${{ matrix.python-version }}-\ - ${{ hashFiles('**/requirements-test.txt') }}" + ${{ hashFiles('**/requirements-test.txt') }}-\ + ${{ hashFiles('**/requirements.txt') }}" restore-keys: | test-${{ runner.os }}-py${{ matrix.python-version }}- test-${{ runner.os }}- diff --git a/requirements.txt b/requirements.txt index d08662f..4ed536d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ --e . +--editable . wheel From 77edca80588dc3230c11af242e5996450517da25 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 4 Sep 2020 17:31:20 -0400 Subject: [PATCH 169/282] Update action/upload-artifact version we use to v2 --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 09b855a..c90c0c0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -102,7 +102,7 @@ jobs: - name: Build artifacts run: python3 setup.py sdist bdist_wheel - name: Upload artifacts - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v2 with: name: dist-${{ matrix.python-version }} path: dist From 5d92f68c57d62e899decaecec0ddb53a3452e72e Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 28 Sep 2020 16:47:52 -0400 Subject: [PATCH 170/282] Add support to run this package as a module Also use the long form of -e in requirements-test.txt. --- requirements-test.txt | 2 +- src/example/__main__.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 src/example/__main__.py diff --git a/requirements-test.txt b/requirements-test.txt index 0a6d7d6..1d0a5a4 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,2 +1,2 @@ +--editable .[test] --requirement requirements.txt --e .[test] diff --git a/src/example/__main__.py b/src/example/__main__.py new file mode 100644 index 0000000..11a3238 --- /dev/null +++ b/src/example/__main__.py @@ -0,0 +1,5 @@ +"""Code to run if this package is used as a Python module.""" + +from .example import main + +main() From b072dc459fbc6b8aab1c4691e23c7a0f7b168559 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 6 Oct 2020 14:04:11 -0400 Subject: [PATCH 171/282] Change execution status of example.py Remove the execute bit in file permissions and remove the shebang line. Since this is configured as a Python package that is installed using pip, it should only use `python -m example` or the `example` console script to be run. --- src/example/example.py | 2 -- 1 file changed, 2 deletions(-) mode change 100755 => 100644 src/example/example.py diff --git a/src/example/example.py b/src/example/example.py old mode 100755 new mode 100644 index 556e8b5..73faa33 --- a/src/example/example.py +++ b/src/example/example.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - """example is an example Python library and tool. Divide one integer by another and log the result. Also log some information From 5c83a56a947a559f6e76b44b7fbaff5190e377e7 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 6 Oct 2020 16:23:17 -0400 Subject: [PATCH 172/282] Add support for Python 3.9 --- .github/workflows/build.yml | 4 ++-- setup.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fe25063..dcb7e02 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -48,7 +48,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8] + python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 @@ -82,7 +82,7 @@ jobs: needs: [lint, test] strategy: matrix: - python-version: [3.6, 3.7, 3.8] + python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 diff --git a/setup.py b/setup.py index ac0d34e..e5122c3 100644 --- a/setup.py +++ b/setup.py @@ -62,6 +62,7 @@ def package_vars(version_file): "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", ], python_requires=">=3.6", # What does your project relate to? From 6a4164dfbac372fde6babd98ece2ab61c9b5e380 Mon Sep 17 00:00:00 2001 From: Felddy Date: Wed, 7 Oct 2020 12:41:47 -0400 Subject: [PATCH 173/282] Bump build workflow python-version to 3.9 --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7a92841..3493928 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.9 - name: Store installed Python version run: | echo "::set-env name=PY_VERSION::"\ From c9c00f1d03d2185dfa4a7cab0ec45bd76387c95a Mon Sep 17 00:00:00 2001 From: Felddy Date: Wed, 7 Oct 2020 12:42:59 -0400 Subject: [PATCH 174/282] Replace deprecated set-env workflow syntax See: https://github.blog/changelog/2020-10-01-github-actions-deprecating-set-env-and-add-path-commands/ --- .github/workflows/build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3493928..d0563b8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,8 +21,9 @@ jobs: python-version: 3.9 - name: Store installed Python version run: | - echo "::set-env name=PY_VERSION::"\ - "$(python -c "import platform;print(platform.python_version())")" + echo "PY_VERSION="\ + "$(python -c "import platform;print(platform.python_version())")" \ + >> $GITHUB_ENV - name: Cache linting environments uses: actions/cache@v2 with: From 7389a1b0cfa7e6d1b50fccef2381d3c28d31e49c Mon Sep 17 00:00:00 2001 From: Felddy Date: Wed, 7 Oct 2020 12:43:54 -0400 Subject: [PATCH 175/282] Enable weekly dependabot scans of github-actions and pip --- .github/dependabot.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..1fd2ed3 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +--- + +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" From d79989a34e4d39fea0cd2eafbc37f17c0cde992b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 7 Oct 2020 15:12:16 -0400 Subject: [PATCH 176/282] Add testing for __main__.py in pursuit of glorious 100% coverage --- tests/test_example.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_example.py b/tests/test_example.py index 689385d..ba79ccf 100644 --- a/tests/test_example.py +++ b/tests/test_example.py @@ -44,6 +44,18 @@ def test_stdout_version(capsys): ), "standard output by '--version' should agree with module.__version__" +def test___main__(capsys): + """Verify that the __main__.py file loads correctly.""" + with pytest.raises(SystemExit): + with patch.object(sys, "argv", ["bogus", "--version"]): + # cisagov Libraries + import example.__main__ # noqa: F401 + captured = capsys.readouterr() + assert ( + captured.out == f"{PROJECT_VERSION}\n" + ), "standard output by '--version' should agree with module.__version__" + + @pytest.mark.skipif( RELEASE_TAG in [None, ""], reason="this is not a release (RELEASE_TAG not set)" ) From db59fc1e6bd1d8937b7eb0c5c9a2e604764e9b19 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 8 Oct 2020 15:08:24 -0400 Subject: [PATCH 177/282] Enable CodeQL scanning for Python. See: https://securitylab.github.com/tools/codeql --- .github/workflows/codeql-analysis.yml | 76 +++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..2efa411 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,76 @@ +--- +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +name: "CodeQL" + +on: + push: + # Push on all branches + # branches: [develop] + pull_request: + # The branches below must be a subset of the branches above + branches: [develop] + schedule: + - cron: '0 14 * * 6' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + # Override automatic language detection by changing the below list + # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', + # 'python'] + language: ['python'] + # Learn more... + # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + # We must fetch at least the immediate parents so that if this is + # a pull request then we can checkout the head. + fetch-depth: 2 + + # If this run was triggered by a pull request event, then checkout + # the head of the pull request instead of the merge commit. + - run: git checkout HEAD^2 + if: ${{ github.event_name == 'pull_request' }} + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a + # config file. By default, queries listed here will override any + # specified in a config file. Prefix the list here with "+" to use + # these queries and those in the config file. queries: + # ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or + # Java). If this step fails, then you should remove it and run the build + # manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏️ If the Autobuild fails above, remove it and uncomment the following + # three lines and modify them (or add more) to build your code if your + # project uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 From e14ecde029ecfbc39759055f7c03d7bc4ac7f458 Mon Sep 17 00:00:00 2001 From: Hillary Date: Fri, 9 Oct 2020 10:07:04 -0400 Subject: [PATCH 178/282] Add link to how to set up a new repo with skeletons --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 606df0a..66e6444 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,10 @@ appropriate for the major languages that we use. In many cases you will instead want to use one of the more specific skeleton projects derived from this one. +## New Repositories from a Skeleton ## + +Please see our [Project Setup guide](https://github.com/cisagov/development-guide/tree/develop/project_setup) for step-by-step instructions on how to start a new repository from a skeleton. This will save you time and effort when configuring a new repository! + ## Contributing ## We welcome contributions! Please see [here](CONTRIBUTING.md) for From 57d927d7d17f9b41aa49e5a57173005eb69719c9 Mon Sep 17 00:00:00 2001 From: Hillary Date: Fri, 9 Oct 2020 10:10:12 -0400 Subject: [PATCH 179/282] Add line breaks for linting Pre-commit is now installed properly. --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 66e6444..83da801 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,10 @@ skeleton projects derived from this one. ## New Repositories from a Skeleton ## -Please see our [Project Setup guide](https://github.com/cisagov/development-guide/tree/develop/project_setup) for step-by-step instructions on how to start a new repository from a skeleton. This will save you time and effort when configuring a new repository! +Please see our [Project Setup guide](https://github.com/cisagov/development-guide/tree/develop/project_setup) +for step-by-step instructions on how to start a new repository from +a skeleton. This will save you time and effort when configuring a +new repository! ## Contributing ## From b5ef97dd4bfb008e08c5abbc0e18848798b1c803 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 19 Oct 2020 11:49:41 -0400 Subject: [PATCH 180/282] Remove seed-isort-config pre-commit hook As of isort >=5, which we use, the seed-isort-config hook has been deprecated, and the repository archived. I have removed the hook and the configuration options it updated as a result. --- .isort.cfg | 5 ----- .pre-commit-config.yaml | 4 ---- 2 files changed, 9 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index 2fd6b08..46d45f3 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -6,10 +6,5 @@ import_heading_stdlib=Standard Python Libraries import_heading_thirdparty=Third-Party Libraries import_heading_firstparty=cisagov Libraries -# Should be auto-populated by seed-isort-config hook -known_third_party= -# These must be manually set to correctly separate them from third party libraries -known_first_party= - # Run isort under the black profile to align with our other Python linting profile=black diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2689bb0..84fbb49 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -60,10 +60,6 @@ repos: rev: 20.8b1 hooks: - id: black - - repo: https://github.com/asottile/seed-isort-config - rev: v2.2.0 - hooks: - - id: seed-isort-config - repo: https://github.com/timothycrosley/isort rev: 5.5.0 hooks: From de409db3f7567c342f6942ffef9a98dded20ede0 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 19 Oct 2020 11:52:09 -0400 Subject: [PATCH 181/282] Use local filename in README link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 606df0a..b445f38 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ skeleton projects derived from this one. ## Contributing ## -We welcome contributions! Please see [here](CONTRIBUTING.md) for +We welcome contributions! Please see [`CONTRIBUTING.md`](CONTRIBUTING.md) for details. ## License ## From 59eef4cc771b2340e131b216e4417b822e8d0b3f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 19 Oct 2020 11:53:09 -0400 Subject: [PATCH 182/282] Run pre-commit autoupdate --- .pre-commit-config.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 84fbb49..6ab5b5b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,13 +27,13 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.23.2 + rev: v0.24.0 hooks: - id: markdownlint args: - --config=.mdl_config.json - repo: https://github.com/adrienverge/yamllint - rev: v1.24.2 + rev: v1.25.0 hooks: - id: yamllint - repo: https://github.com/detailyang/pre-commit-shell @@ -41,7 +41,7 @@ repos: hooks: - id: shell-lint - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.3 + rev: 3.8.4 hooks: - id: flake8 additional_dependencies: @@ -61,16 +61,16 @@ repos: hooks: - id: black - repo: https://github.com/timothycrosley/isort - rev: 5.5.0 + rev: 5.6.4 hooks: - id: isort - repo: https://github.com/ansible/ansible-lint.git - rev: v4.3.4 + rev: v4.3.5 hooks: - id: ansible-lint # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git - rev: v1.37.0 + rev: v1.43.0 hooks: - id: terraform_fmt # There are ongoing issues with how this command works. This issue @@ -94,10 +94,10 @@ repos: hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier - rev: 2.1.1 + rev: 2.1.2 hooks: - id: prettier - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.782 + rev: v0.790 hooks: - id: mypy From 73a7dcda49e2fc700c3435f110fd1018432a28ca Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 22 Oct 2020 12:48:42 -0400 Subject: [PATCH 183/282] Add explanation for the noqa used in one of the tests --- tests/test_example.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_example.py b/tests/test_example.py index ba79ccf..914ea63 100644 --- a/tests/test_example.py +++ b/tests/test_example.py @@ -48,6 +48,11 @@ def test___main__(capsys): """Verify that the __main__.py file loads correctly.""" with pytest.raises(SystemExit): with patch.object(sys, "argv", ["bogus", "--version"]): + # F401 is a "Module imported but unused" warning. This import + # emulates how this project would be run as a module. The only thing + # being done by __main__ is importing the main entrypoint of the + # package and running it, so there is nothing to use from this + # import. As a result, we can safely ignore this warning. # cisagov Libraries import example.__main__ # noqa: F401 captured = capsys.readouterr() From 72715e3d288fe50f083b82cae68f01eed7db803e Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 22 Oct 2020 12:50:01 -0400 Subject: [PATCH 184/282] Change a test name to be more descriptive --- tests/test_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_example.py b/tests/test_example.py index 914ea63..3a22848 100644 --- a/tests/test_example.py +++ b/tests/test_example.py @@ -44,7 +44,7 @@ def test_stdout_version(capsys): ), "standard output by '--version' should agree with module.__version__" -def test___main__(capsys): +def test_running_as_module(capsys): """Verify that the __main__.py file loads correctly.""" with pytest.raises(SystemExit): with patch.object(sys, "argv", ["bogus", "--version"]): From 919460cb213ac2e3c507f1c2fb4c8e7cff54783b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 22 Oct 2020 13:42:10 -0400 Subject: [PATCH 185/282] Change repo for the prettier pre-commit hook Per https://github.com/prettier/prettier/issues/9459 and specifically https://github.com/prettier/prettier/issues/9459#issuecomment-713223710 the prettier hook for pre-commit is being broken out into its own repo. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6ab5b5b..7362f89 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -93,8 +93,8 @@ repos: rev: v2.0.0 hooks: - id: docker-compose-check - - repo: https://github.com/prettier/prettier - rev: 2.1.2 + - repo: https://github.com/prettier/pre-commit + rev: v2.1.2 hooks: - id: prettier - repo: https://github.com/pre-commit/mirrors-mypy From a842abbfefefdb03bd1661dfb5956d7b09eeae25 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 3 Nov 2020 12:40:54 -0500 Subject: [PATCH 186/282] Insist that the cisagov devs are the owners of the .github directory This additional clause must remain at the _end_ of the CODEOWNERS file so that it cannot be overridden by a later clause. We want to make it so that all the .github files including CODEOWNERS are protected so only code owners (the dev team) can approve modifications to them. This will prevent configuration changes from breaking Actions and other management-type functions that the files in this directory control. By setting the .github files/folder to require code owner approval for changes, workflow and management changes will require dev team review and checking. Resolves #56. --- .github/CODEOWNERS | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 7735a52..5671d70 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,7 +1,10 @@ # Each line is a file pattern followed by one or more owners. -# These owners will be the default owners for everything in -# the repo. Unless a later match takes precedence, -# these owners will be requested for review when someone -# opens a pull request. +# These owners will be the default owners for everything in the +# repo. Unless a later match takes precedence, these owners will be +# requested for review when someone opens a pull request. * @dav3r @felddy @hillaryj @jsf9k @mcdonnnj + +# These folks own any files in the /.github directory at the root of +# the repository and any of its subdirectories. +/.github/ @dav3r @felddy @hillaryj @jsf9k @mcdonnnj From 5b199bc86974d09044df2e510d7ac4c6ba4ad850 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 3 Nov 2020 14:51:18 -0500 Subject: [PATCH 187/282] Remove offending slash Thanks to @dav3r and @mcdonnnj for the suggestion. --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5671d70..9c3d21f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,6 +5,6 @@ # requested for review when someone opens a pull request. * @dav3r @felddy @hillaryj @jsf9k @mcdonnnj -# These folks own any files in the /.github directory at the root of +# These folks own any files in the .github directory at the root of # the repository and any of its subdirectories. /.github/ @dav3r @felddy @hillaryj @jsf9k @mcdonnnj From 02c5a6f8bf72e7695cc5957cd62a7e137f55f80e Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 10 Nov 2020 22:10:38 -0500 Subject: [PATCH 188/282] Use the python version output by actions/setup-python There is no need to run python code to determine the python version. Resolves #58. See here for details: https://github.com/actions/setup-python/blob/main/action.yml#L14-L16 --- .github/workflows/build.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d0563b8..bae558f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,26 +16,25 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - id: setup-python + uses: actions/setup-python@v2 with: python-version: 3.9 - - name: Store installed Python version - run: | - echo "PY_VERSION="\ - "$(python -c "import platform;print(platform.python_version())")" \ - >> $GITHUB_ENV - name: Cache linting environments uses: actions/cache@v2 with: path: | ${{ env.PIP_CACHE_DIR }} ${{ env.PRE_COMMIT_CACHE_DIR }} - key: "lint-${{ runner.os }}-py${{ env.PY_VERSION }}-\ + key: | + lint-${{ runner.os }}-\ + py${{ steps.setup-python.outputs.python-version }}-\ ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/requirements.txt') }}-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" restore-keys: | - lint-${{ runner.os }}-py${{ env.PY_VERSION }}- + lint-${{ runner.os }}-\ + py${{ steps.setup-python.outputs.python-version }}- lint-${{ runner.os }}- - name: Install dependencies run: | From a9ee3dd35081e5f589e9e14479391c986161384e Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 13 Nov 2020 13:37:58 -0500 Subject: [PATCH 189/282] Make sure the requirements in setup.py are sorted Sort the install_requires and extras_require["test"] arguments by key. --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index e5122c3..25a86b7 100644 --- a/setup.py +++ b/setup.py @@ -72,10 +72,10 @@ def package_vars(version_file): package_data={"example": ["data/*.txt"]}, py_modules=[splitext(basename(path))[0] for path in glob("src/*.py")], include_package_data=True, - install_requires=["docopt", "setuptools >= 24.2.0", "schema"], + install_requires=["docopt", "schema", "setuptools >= 24.2.0"], extras_require={ "test": [ - "pre-commit", + "coverage", # coveralls 1.11.0 added a service number for calls from # GitHub Actions. This caused a regression which resulted in a 422 # response from the coveralls API with the message: @@ -83,7 +83,7 @@ def package_vars(version_file): # 1.11.1 fixed this issue, but to ensure expected behavior we'll pin # to never grab the regression version. "coveralls != 1.11.0", - "coverage", + "pre-commit", "pytest-cov", "pytest", ] From 64c3fb61ab1eb832821d57882f42407e88970740 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sun, 15 Nov 2020 23:02:59 -0500 Subject: [PATCH 190/282] Organize pre-commit hooks Group related pre-commit hooks together. Make sure that hooks are alphabetically sorted within those groups. --- .pre-commit-config.yaml | 48 +++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7362f89..e9dde69 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,30 +26,30 @@ repos: - --autofix - id: requirements-txt-fixer - id: trailing-whitespace + + # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli rev: v0.24.0 hooks: - id: markdownlint args: - --config=.mdl_config.json + - repo: https://github.com/prettier/pre-commit + rev: v2.1.2 + hooks: + - id: prettier - repo: https://github.com/adrienverge/yamllint rev: v1.25.0 hooks: - id: yamllint + + # Shell script hooks - repo: https://github.com/detailyang/pre-commit-shell rev: 1.0.5 hooks: - id: shell-lint - - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.4 - hooks: - - id: flake8 - additional_dependencies: - - flake8-docstrings - - repo: https://github.com/asottile/pyupgrade - rev: v2.7.2 - hooks: - - id: pyupgrade + + # Python hooks - repo: https://github.com/PyCQA/bandit rev: 1.6.2 hooks: @@ -60,15 +60,33 @@ repos: rev: 20.8b1 hooks: - id: black + - repo: https://gitlab.com/pycqa/flake8 + rev: 3.8.4 + hooks: + - id: flake8 + additional_dependencies: + - flake8-docstrings - repo: https://github.com/timothycrosley/isort rev: 5.6.4 hooks: - id: isort + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.790 + hooks: + - id: mypy + - repo: https://github.com/asottile/pyupgrade + rev: v2.7.2 + hooks: + - id: pyupgrade + + # Ansible hooks - repo: https://github.com/ansible/ansible-lint.git rev: v4.3.5 hooks: - id: ansible-lint # files: molecule/default/playbook.yml + + # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform.git rev: v1.43.0 hooks: @@ -89,15 +107,9 @@ repos: # above have been resolved, which we hope will be with the release of # Terraform 0.13. # - id: terraform_validate + + # Docker hooks - repo: https://github.com/IamTheFij/docker-pre-commit rev: v2.0.0 hooks: - id: docker-compose-check - - repo: https://github.com/prettier/pre-commit - rev: v2.1.2 - hooks: - - id: prettier - - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.790 - hooks: - - id: mypy From ede0485c141b24d09d0c8facc2d6f94eb33175a3 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 16 Nov 2020 00:10:02 -0500 Subject: [PATCH 191/282] Change how the version number is extracted in setup.py Switch to the file line reading version of extracting the version from https://packaging.python.org/guides/single-sourcing-package-version/ instead of the exec method on the same page. The exec method required us to use a "# nosec" to manually disable Bandit checking on that line. Although that method is more straightforward, I do not feel that it is worth using an exec in the codebase when another option is available. --- setup.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index e5122c3..99ef5c6 100644 --- a/setup.py +++ b/setup.py @@ -9,8 +9,9 @@ """ # Standard Python Libraries +import codecs from glob import glob -from os.path import basename, splitext +from os.path import abspath, basename, dirname, join, splitext # Third-Party Libraries from setuptools import find_packages, setup @@ -22,18 +23,29 @@ def readme(): return f.read() -def package_vars(version_file): - """Read in and return the variables defined by the version_file.""" - pkg_vars = {} - with open(version_file) as f: - exec(f.read(), pkg_vars) # nosec - return pkg_vars +# Below two methods were pulled from: +# https://packaging.python.org/guides/single-sourcing-package-version/ +def read(rel_path): + """Open a file for reading from a given relative path.""" + here = abspath(dirname(__file__)) + with codecs.open(join(here, rel_path), "r") as fp: + return fp.read() + + +def get_version(version_file): + """Extract a version number from the given file path.""" + for line in read(version_file).splitlines(): + if line.startswith("__version__"): + delim = '"' if '"' in line else "'" + return line.split(delim)[1] + else: + raise RuntimeError("Unable to find version string.") setup( name="example", # Versions should comply with PEP440 - version=package_vars("src/example/_version.py")["__version__"], + version=get_version("src/example/_version.py"), description="Example python library", long_description=readme(), long_description_content_type="text/markdown", From 40b3b80c4d52696810884763c7adcd73ff8e353f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 16 Nov 2020 00:18:21 -0500 Subject: [PATCH 192/282] Add comment explaining a Flake8 check disable --- src/example/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/example/__init__.py b/src/example/__init__.py index cf807c3..98b5e04 100644 --- a/src/example/__init__.py +++ b/src/example/__init__.py @@ -1,4 +1,8 @@ """The example library.""" +# We disable a Flake8 check for "Module imported but unused (F401)" here because +# although this import is not directly used, it populates the value +# package_name.__version__, which is used to get version information about this +# Python package. from ._version import __version__ # noqa: F401 from .example import example_div From 57fc8a4ef7bbb4f2eca4ba930b3307e1c3b50763 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 16 Nov 2020 07:47:18 -0500 Subject: [PATCH 193/282] Resolve LGTM alert --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 99ef5c6..80fc868 100644 --- a/setup.py +++ b/setup.py @@ -38,8 +38,7 @@ def get_version(version_file): if line.startswith("__version__"): delim = '"' if '"' in line else "'" return line.split(delim)[1] - else: - raise RuntimeError("Unable to find version string.") + raise RuntimeError("Unable to find version string.") setup( From f4131e57d81557a751eec90ff0f07784809aa5d2 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 20 Nov 2020 12:22:39 -0500 Subject: [PATCH 194/282] Clean up our actions/cache step Removed name because it was not more informative than the default. Swapped out a hardcoded job reference for the github.job context value. Switch the base cache key to a step environment value so we can set it once and reuse. Removed additional restore-key value that might have undesirable results. --- .github/workflows/build.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bae558f..5c65f71 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,22 +20,20 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.9 - - name: Cache linting environments - uses: actions/cache@v2 + - uses: actions/cache@v2 + env: + BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ + py${{ steps.setup-python.outputs.python-version }}-" with: path: | ${{ env.PIP_CACHE_DIR }} ${{ env.PRE_COMMIT_CACHE_DIR }} - key: | - lint-${{ runner.os }}-\ - py${{ steps.setup-python.outputs.python-version }}-\ + key: "${{ env.BASE_CACHE_KEY }}\ ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/requirements.txt') }}-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" restore-keys: | - lint-${{ runner.os }}-\ - py${{ steps.setup-python.outputs.python-version }}- - lint-${{ runner.os }}- + ${{ env.BASE_CACHE_KEY }} - name: Install dependencies run: | python -m pip install --upgrade pip From 54858fb8b632d9fbf362a225e0a36c7b2e6d2836 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 20 Nov 2020 13:43:28 -0500 Subject: [PATCH 195/282] Update cache key used for test and build jobs Use the full semver for the installed Python version in cache keys. --- .github/workflows/build.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4a074c4..20917fa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,7 +51,8 @@ jobs: python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - id: setup-python + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Cache testing environments @@ -59,12 +60,12 @@ jobs: with: path: ${{ env.PIP_CACHE_DIR }} key: "test-${{ runner.os }}-\ - py${{ matrix.python-version }}-\ + py${{ steps.setup-python.outputs.python-version }}-\ ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/requirements.txt') }}" restore-keys: | test-${{ runner.os }}-\ - py${{ matrix.python-version }}- + py${{ steps.setup-python.outputs.python-version }}- test-${{ runner.os }}- - name: Install dependencies run: | @@ -87,7 +88,8 @@ jobs: python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - id: setup-python + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Cache building environments @@ -95,11 +97,11 @@ jobs: with: path: ${{ env.PIP_CACHE_DIR }} key: "build-${{ runner.os }}-\ - py${{ matrix.python-version }}-\ + py${{ steps.setup-python.outputs.python-version }}-\ ${{ hashFiles('**/requirements.txt') }}" restore-keys: | build-${{ runner.os }}-\ - py${{ matrix.python-version }}- + py${{ steps.setup-python.outputs.python-version }}- build-${{ runner.os }}- - name: Install dependencies run: | From 12e1e2c33183331638918044bf07ab8b84cf155b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 3 Dec 2020 11:50:42 -0500 Subject: [PATCH 196/282] Update test and build workflow jobs Update the test and build workflow jobs to use the actions/cache style that was done for the lint job. --- .github/workflows/build.yml | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 83bf211..40eac3a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -52,18 +52,17 @@ jobs: - uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Cache testing environments - uses: actions/cache@v2 + - uses: actions/cache@v2 + env: + BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ + py${{ matrix.python-version }}-" with: path: ${{ env.PIP_CACHE_DIR }} - key: "test-${{ runner.os }}-\ - py${{ matrix.python-version }}-\ + key: "${{ env.BASE_CACHE_KEY }}\ ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/requirements.txt') }}" restore-keys: | - test-${{ runner.os }}-\ - py${{ matrix.python-version }}- - test-${{ runner.os }}- + ${{ env.BASE_CACHE_KEY }} - name: Install dependencies run: | python -m pip install --upgrade pip @@ -88,17 +87,16 @@ jobs: - uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Cache building environments - uses: actions/cache@v2 + - uses: actions/cache@v2 + env: + BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ + py${{ matrix.python-version }}-" with: path: ${{ env.PIP_CACHE_DIR }} - key: "build-${{ runner.os }}-\ - py${{ matrix.python-version }}-\ + key: "${{ env.BASE_CACHE_KEY }}\ ${{ hashFiles('**/requirements.txt') }}" restore-keys: | - build-${{ runner.os }}-\ - py${{ matrix.python-version }}- - build-${{ runner.os }}- + ${{ env.BASE_CACHE_KEY }} - name: Install dependencies run: | python -m pip install --upgrade pip wheel From b769825c41defa9365c108fbed03e79b753680c1 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 16 Dec 2020 18:03:13 -0500 Subject: [PATCH 197/282] Update prettier hook details Per https://github.com/prettier/prettier/pull/8937 the pre-commit hook has been moved to https://github.com/pre-commit/mirrors-prettier. I have also updated to the latest version in that repository. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e9dde69..1d6845f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,8 +34,8 @@ repos: - id: markdownlint args: - --config=.mdl_config.json - - repo: https://github.com/prettier/pre-commit - rev: v2.1.2 + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v2.2.1 hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint From 2d6bf50eab010ed699f5ed38f91fd4f397c473fd Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 16 Dec 2020 18:13:04 -0500 Subject: [PATCH 198/282] Update black hook repo url At some point the python GitHub organization renamed to psf (Python Software Foundation). Although it redirects with no issue, I think it should be updated to reflect the current repository URL. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1d6845f..af393d0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -56,7 +56,7 @@ repos: - id: bandit args: - --config=.bandit.yml - - repo: https://github.com/python/black + - repo: https://github.com/psf/black rev: 20.8b1 hooks: - id: black From 5115baee52b77770f86f22af8618c432c8f0c064 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 16 Dec 2020 18:15:03 -0500 Subject: [PATCH 199/282] Remove trailing '.git's from repository URLs Two hooks had trailing '.git's in the URLs. Although this is not a problem, we should be consistent in how we format things. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index af393d0..fc1ed74 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -80,14 +80,14 @@ repos: - id: pyupgrade # Ansible hooks - - repo: https://github.com/ansible/ansible-lint.git + - repo: https://github.com/ansible/ansible-lint rev: v4.3.5 hooks: - id: ansible-lint # files: molecule/default/playbook.yml # Terraform hooks - - repo: https://github.com/antonbabenko/pre-commit-terraform.git + - repo: https://github.com/antonbabenko/pre-commit-terraform rev: v1.43.0 hooks: - id: terraform_fmt From a7f86ddad0b4a70d0ee06ce8f6316749cb83e3f9 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 16 Dec 2020 18:21:03 -0500 Subject: [PATCH 200/282] Enable two additional hooks from default pre-commit-hooks Enabled 'check-case-conflict' because of our mixed Linux and macOS development. Although APFS supports case-sensitive containers, it is not the default as far as I am aware. Linux filesystems are typically case-sensitive however. With the merge of https://github.com/cisagov/development-guide/pull/42 we now have a TOML file in a repository, so it does not hurt to add this hook in case more are added in the future. --- .pre-commit-config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fc1ed74..3e2c07c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,9 +7,11 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v3.2.0 hooks: + - id: check-case-conflict - id: check-executables-have-shebangs - id: check-json - id: check-merge-conflict + - id: check-toml - id: check-xml - id: debug-statements - id: detect-aws-credentials From 06159cbb65e0f5bf76ffe487b82684f0fc632f59 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 16 Dec 2020 18:30:17 -0500 Subject: [PATCH 201/282] Run pre-commit autoupdate --- .pre-commit-config.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3e2c07c..cc668e1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.2.0 + rev: v3.4.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs @@ -31,7 +31,7 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.24.0 + rev: v0.26.0 hooks: - id: markdownlint args: @@ -53,7 +53,7 @@ repos: # Python hooks - repo: https://github.com/PyCQA/bandit - rev: 1.6.2 + rev: 1.7.0 hooks: - id: bandit args: @@ -77,20 +77,20 @@ repos: hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v2.7.2 + rev: v2.7.4 hooks: - id: pyupgrade # Ansible hooks - repo: https://github.com/ansible/ansible-lint - rev: v4.3.5 + rev: v4.3.7 hooks: - id: ansible-lint # files: molecule/default/playbook.yml # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.43.0 + rev: v1.45.0 hooks: - id: terraform_fmt # There are ongoing issues with how this command works. This issue From d836e91d323472fbdab0becb78bac7c79e3a2e73 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 17 Dec 2020 13:50:34 -0500 Subject: [PATCH 202/282] Update repo URL for the isort hook I missed that this repository was transfered from the creator, Timothy Crosley, to the PyCQA organization. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cc668e1..e042de8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -68,7 +68,7 @@ repos: - id: flake8 additional_dependencies: - flake8-docstrings - - repo: https://github.com/timothycrosley/isort + - repo: https://github.com/PyCQA/isort rev: 5.6.4 hooks: - id: isort From 46ec3e7002c98c6653b39595008456eff1cd7f01 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 31 Dec 2020 15:06:18 -0500 Subject: [PATCH 203/282] Update ansible-lint hook URL --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e042de8..af4fed5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -82,7 +82,7 @@ repos: - id: pyupgrade # Ansible hooks - - repo: https://github.com/ansible/ansible-lint + - repo: https://github.com/ansible-community/ansible-lint rev: v4.3.7 hooks: - id: ansible-lint From 362b054459b15b6dd3ff5d7eb587682fee82f72a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 31 Dec 2020 15:08:07 -0500 Subject: [PATCH 204/282] Run pre-commit autoupdate --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index af4fed5..2b3a71e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -69,7 +69,7 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/PyCQA/isort - rev: 5.6.4 + rev: 5.7.0 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy From 8f7435a6fa7494c380b7b45831c7dec5ce3cd2bc Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 31 Dec 2020 15:09:53 -0500 Subject: [PATCH 205/282] Add the pre-commit-packer repo and hooks --- .pre-commit-config.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e9dde69..28aa9bd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -113,3 +113,10 @@ repos: rev: v2.0.0 hooks: - id: docker-compose-check + + # Packer hooks + - repo: https://github.com/cisagov/pre-commit-packer + rev: v0.0.2 + hooks: + - id: packer_validate + - id: packer_fmt From 2fb4e15cf6c743a3ce0ab434d3de7ae62ffc6fe2 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 7 Jan 2021 04:07:38 -0500 Subject: [PATCH 206/282] Add the --strict flag to the yamllint pre-commot hook The --strict flag will cause the yamllint hook to exit with a non-zero exit code when warnings are found instead of only when errors are found. --- .pre-commit-config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e9dde69..26c06a6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,6 +42,8 @@ repos: rev: v1.25.0 hooks: - id: yamllint + args: + - --strict # Shell script hooks - repo: https://github.com/detailyang/pre-commit-shell From ecdfc527d4338218f005684439214c2d126aa11b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 15 Jan 2021 10:34:34 -0500 Subject: [PATCH 207/282] Add the beautysh pre-commit hook This hook performs auto-formatting (beautifying) of Bash scripts. --- .pre-commit-config.yaml | 7 +++++++ setup-env | 14 +++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8a3c7eb..42a2c7a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,6 +48,13 @@ repos: - --strict # Shell script hooks + - repo: https://github.com/lovesegfault/beautysh + rev: 6.0.1 + hooks: + - id: beautysh + args: + - --indent-size + - '2' - repo: https://github.com/detailyang/pre-commit-shell rev: 1.0.5 hooks: diff --git a/setup-env b/setup-env index 4d822c4..1579e04 100755 --- a/setup-env +++ b/setup-env @@ -50,14 +50,14 @@ while (( "$#" )); do shift ;; -*) # unsupported flags - echo "Error: Unsupported flag $1" >&2 - exit 1 - ;; + echo "Error: Unsupported flag $1" >&2 + exit 1 + ;; *) # preserve positional arguments - PARAMS="$PARAMS $1" - shift - ;; - esac + PARAMS="$PARAMS $1" + shift + ;; + esac done # set positional arguments in their proper place From 12033ed60cff74437014a4f1cf9cde9d2b4a13ef Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 5 Feb 2021 13:19:51 -0500 Subject: [PATCH 208/282] Add the mypy package to dev rquirements --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index cb51627..1d7e302 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ --requirement requirements-test.txt ipython +mypy semver From 6bd93ff6fbd1e9fe1a6e508d6a65225b63220a53 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sun, 31 Jan 2021 19:24:54 -0500 Subject: [PATCH 209/282] Update pre-commit hooks with `pre-commit autoupdate` --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8a3c7eb..e1da8e2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,7 +41,7 @@ repos: hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint - rev: v1.25.0 + rev: v1.26.0 hooks: - id: yamllint args: @@ -75,17 +75,17 @@ repos: hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.790 + rev: v0.800 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v2.7.4 + rev: v2.10.0 hooks: - id: pyupgrade # Ansible hooks - repo: https://github.com/ansible-community/ansible-lint - rev: v4.3.7 + rev: v5.0.0 hooks: - id: ansible-lint # files: molecule/default/playbook.yml From b54a0fc817545941b6cb0707fe127e79f5c06f26 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 11 Feb 2021 18:27:08 -0500 Subject: [PATCH 210/282] Update the URL used for our Python packages us-cert.gov now forwards to us-cert.cisa.gov, and since NCATS is a pre-CISA designation it makes sense to update this URL. The URL points to a specific set of services offered that does not encompass everything that is hosted in the cisagov org on GitHub. We have chosen the page describing CISA's cybersecurity mission as the best replacement URL. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index a458722..56b9b95 100644 --- a/setup.py +++ b/setup.py @@ -48,8 +48,8 @@ def get_version(version_file): description="Example python library", long_description=readme(), long_description_content_type="text/markdown", - # NCATS "homepage" - url="https://www.us-cert.gov/resources/ncats", + # Landing page for CISA's cybersecurity mission + url="https://www.cisa.gov/cybersecurity", # The project's main homepage download_url="https://github.com/cisagov/skeleton-python-library", # Author details From 7409f9d3536e81a872cb67c02749c2da0864032c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 11 Feb 2021 18:32:18 -0500 Subject: [PATCH 211/282] Update the author_email field in package metadata We no longer have access to the old email address. Additionally the agency has gotten it's own email domain, so we prefer cisa.dhs.gov addresses at this point. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 56b9b95..4a62591 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ def get_version(version_file): download_url="https://github.com/cisagov/skeleton-python-library", # Author details author="Cyber and Infrastructure Security Agency", - author_email="ncats@hq.dhs.gov", + author_email="github@cisa.dhs.gov", license="License :: CC0 1.0 Universal (CC0 1.0) Public Domain Dedication", # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ From 19e1fb997c3fae8eee180c8c99a0229d05cb3e6f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 11 Feb 2021 18:36:57 -0500 Subject: [PATCH 212/282] Remove download_url field and add project_urls field to package metadata The download_urls field was being used incorrectly, as it is supposed to provide a download URL to a file usable to install the package. This would typically be a tarball of some kind. Additionaly this field is discouraged per https://github.com/pypa/packaging.python.org/issues/293, so we should stop using it in our package metadata. The best replacement I found is the project_urls field, which allows us to provide a number of purpose specific URLs about the package. --- setup.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 4a62591..bdee06b 100644 --- a/setup.py +++ b/setup.py @@ -50,8 +50,12 @@ def get_version(version_file): long_description_content_type="text/markdown", # Landing page for CISA's cybersecurity mission url="https://www.cisa.gov/cybersecurity", - # The project's main homepage - download_url="https://github.com/cisagov/skeleton-python-library", + # Additional URLs for this project per + # https://packaging.python.org/guides/distributing-packages-using-setuptools/#project-urls + project_urls={ + "Source": "https://github.com/cisagov/skeleton-python-library", + "Tracker": "https://github.com/cisagov/skeleton-python-library/issues", + }, # Author details author="Cyber and Infrastructure Security Agency", author_email="github@cisa.dhs.gov", From 7f968776c38b1e8bb8d41aafb71c0445d982482c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 11 Feb 2021 18:51:08 -0500 Subject: [PATCH 213/282] Fix capitalization of Python in description metadata --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bdee06b..45ed1a0 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ def get_version(version_file): name="example", # Versions should comply with PEP440 version=get_version("src/example/_version.py"), - description="Example python library", + description="Example Python library", long_description=readme(), long_description_content_type="text/markdown", # Landing page for CISA's cybersecurity mission From de06bbc81fac2012ef040f61f4ccede70f27e239 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 11 Feb 2021 18:52:56 -0500 Subject: [PATCH 214/282] Fix agency name in author metadata --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 45ed1a0..14bd3fd 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ def get_version(version_file): "Tracker": "https://github.com/cisagov/skeleton-python-library/issues", }, # Author details - author="Cyber and Infrastructure Security Agency", + author="Cybersecurity and Infrastructure Security Agency", author_email="github@cisa.dhs.gov", license="License :: CC0 1.0 Universal (CC0 1.0) Public Domain Dedication", # See https://pypi.python.org/pypi?%3Aaction=list_classifiers From c97a883ada7ae0cbd50f790d4deb4e79639736dd Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 16 Feb 2021 13:18:08 -0500 Subject: [PATCH 215/282] Revert ansible-lint version update The v5 release of ansible-lint introduces breaking changes, so we are holding off on updating until things are resolved. Conversation about this can be tracked in https://github.com/cisagov/skeleton-ansible-role/issues/69. --- .pre-commit-config.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e1da8e2..4fdfdff 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -85,7 +85,9 @@ repos: # Ansible hooks - repo: https://github.com/ansible-community/ansible-lint - rev: v5.0.0 + # This is intentionally being held back because of issues in v5 per + # https://github.com/cisagov/skeleton-ansible-role/issues/69 + rev: v4.3.7 hooks: - id: ansible-lint # files: molecule/default/playbook.yml From 4d8089ed42875c300c29dab9d4ad134ed46d0f2c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 16 Feb 2021 17:49:46 -0500 Subject: [PATCH 216/282] Autoformat bump_version.sh with beautysh --- bump_version.sh | 66 ++++++++++++++++++++++++------------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/bump_version.sh b/bump_version.sh index 861eed0..c0bf25c 100755 --- a/bump_version.sh +++ b/bump_version.sh @@ -14,38 +14,38 @@ old_version=$(sed -n "s/^__version__ = \"\(.*\)\"$/\1/p" $VERSION_FILE) if [ $# -ne 1 ] then - echo "$HELP_INFORMATION" + echo "$HELP_INFORMATION" else - case $1 in - major|minor|patch|prerelease|build) - new_version=$(python -c "import semver; print(semver.bump_$1('$old_version'))") - echo Changing version from "$old_version" to "$new_version" - # A temp file is used to provide compatability with macOS development - # as a result of macOS using the BSD version of sed - tmp_file=/tmp/version.$$ - sed "s/$old_version/$new_version/" $VERSION_FILE > $tmp_file - mv $tmp_file $VERSION_FILE - git add $VERSION_FILE - git commit -m"Bump version from $old_version to $new_version" - git push - ;; - finalize) - new_version=$(python -c "import semver; print(semver.finalize_version('$old_version'))") - echo Changing version from "$old_version" to "$new_version" - # A temp file is used to provide compatability with macOS development - # as a result of macOS using the BSD version of sed - tmp_file=/tmp/version.$$ - sed "s/$old_version/$new_version/" $VERSION_FILE > $tmp_file - mv $tmp_file $VERSION_FILE - git add $VERSION_FILE - git commit -m"Bump version from $old_version to $new_version" - git push - ;; - show) - echo "$old_version" - ;; - *) - echo "$HELP_INFORMATION" - ;; - esac + case $1 in + major|minor|patch|prerelease|build) + new_version=$(python -c "import semver; print(semver.bump_$1('$old_version'))") + echo Changing version from "$old_version" to "$new_version" + # A temp file is used to provide compatability with macOS development + # as a result of macOS using the BSD version of sed + tmp_file=/tmp/version.$$ + sed "s/$old_version/$new_version/" $VERSION_FILE > $tmp_file + mv $tmp_file $VERSION_FILE + git add $VERSION_FILE + git commit -m"Bump version from $old_version to $new_version" + git push + ;; + finalize) + new_version=$(python -c "import semver; print(semver.finalize_version('$old_version'))") + echo Changing version from "$old_version" to "$new_version" + # A temp file is used to provide compatability with macOS development + # as a result of macOS using the BSD version of sed + tmp_file=/tmp/version.$$ + sed "s/$old_version/$new_version/" $VERSION_FILE > $tmp_file + mv $tmp_file $VERSION_FILE + git add $VERSION_FILE + git commit -m"Bump version from $old_version to $new_version" + git push + ;; + show) + echo "$old_version" + ;; + *) + echo "$HELP_INFORMATION" + ;; + esac fi From e59eb44ced5ca2ce1551f5ad04d1c03799f2c353 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 26 Feb 2021 14:33:22 -0500 Subject: [PATCH 217/282] Update CodeQL workflow for GitHub Actions I noticed the following warning when looking at GHA workflow runs: 1 issue was detected with this workflow: git checkout HEAD^2 is no longer necessary. Please remove this step as Code Scanning recommends analyzing the merge commit for best results. After looking into it I updated the workflow per this page on GitHub Docs: https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/troubleshooting-the-codeql-workflow#warning-git-checkout-head2-is-no-longer-necessary --- .github/workflows/codeql-analysis.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 97aff7f..d599e48 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -34,15 +34,6 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v2 - with: - # We must fetch at least the immediate parents so that if this is - # a pull request then we can checkout the head. - fetch-depth: 2 - - # If this run was triggered by a pull request event, then checkout - # the head of the pull request instead of the merge commit. - - run: git checkout HEAD^2 - if: ${{ github.event_name == 'pull_request' }} # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL From 3f80d96ea83b6328ccafa5960807d66b9b58b4fc Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sat, 3 Apr 2021 17:34:19 -0400 Subject: [PATCH 218/282] Use secrets.GITHUB_TOKEN for coveralls in GHA workflow coveralls-python has better integration with running in GitHub Actions now, so we can use the automatically generated GITHUB_TOKEN secret instead of manually setting a COVERALLS_REPO_TOKEN secret for each repository. --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0134014..02ea543 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -75,7 +75,8 @@ jobs: - name: Upload coverage report run: coveralls env: - COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} + COVERALLS_SERVICE_NAME: github + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} if: success() build: runs-on: ubuntu-latest From 30a1c14cebf6cb74b2270e8bf9aa4ec829c99645 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sat, 3 Apr 2021 17:38:42 -0400 Subject: [PATCH 219/282] Update coveralls GHA configuration for parallelization We already ran our tests in parallel for each major version of Python a project supports. This change should better differentiate each test job run in the reports submitted. --- .github/workflows/build.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 02ea543..695e8cc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -75,9 +75,39 @@ jobs: - name: Upload coverage report run: coveralls env: + COVERALLS_FLAG_NAME: "py${{ matrix.python-version }}" + COVERALLS_PARALLEL: true COVERALLS_SERVICE_NAME: github GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} if: success() + coveralls-finish: + runs-on: ubuntu-latest + needs: test + steps: + - uses: actions/checkout@v2 + - id: setup-python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - uses: actions/cache@v2 + env: + BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ + py${{ steps.setup-python.outputs.python-version }}-" + with: + path: ${{ env.PIP_CACHE_DIR }} + key: "${{ env.BASE_CACHE_KEY }}\ + ${{ hashFiles('**/requirements-test.txt') }}-\ + ${{ hashFiles('**/requirements.txt') }}" + restore-keys: | + ${{ env.BASE_CACHE_KEY }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade --requirement requirements-test.txt + - name: Finished coveralls reports + run: coveralls --finish + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} build: runs-on: ubuntu-latest needs: [lint, test] From 5656676bca1f0b68709c9631023f0298a7f7f79c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 7 Apr 2021 16:53:27 -0400 Subject: [PATCH 220/282] Skip CodeQL workflow for Dependabot branches on push events This should resolve the following error: Error: Workflows triggered by Dependabot on the "push" event run with read-only access. Uploading Code Scanning results requires write access. To use Code Scanning with Dependabot, please ensure you are using the "pull_request" event for this workflow and avoid triggering on the "push" event for Dependabot branches. See https://docs.github.com/en/code-security/secure-coding/configuring-code-scanning#scanning-on-push for more information on how to configure these events. --- .github/workflows/codeql-analysis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d599e48..b880c44 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -8,8 +8,9 @@ name: "CodeQL" on: push: - # Push on all branches - # branches: [develop] + # Dependabot triggered push events have read-only access, but uploading code + # scanning requires write access. + branches-ignore: [dependabot/**] pull_request: # The branches below must be a subset of the branches above branches: [develop] From ba49077ab48d1ac7ad45b2bb05d1db346950284b Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 12 Apr 2021 10:37:49 -0400 Subject: [PATCH 221/282] Improve gitignore file comments and organization. * Add a comment describing what the files does. * Add sections to organize patterns. --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 4b15d77..937e21d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ +# This file specifies intentionally untracked files that Git should ignore. +# Files already tracked by Git are not affected. +# See: https://git-scm.com/docs/gitignore + +## Python ## __pycache__ .mypy_cache .python-version From 395279961abae31c75849146d67c785849d8de94 Mon Sep 17 00:00:00 2001 From: Hillary Jeffrey Date: Thu, 15 Apr 2021 10:50:55 -0400 Subject: [PATCH 222/282] Adds a comment to direct additional requirements into setup.py --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 4ed536d..8b75fe9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ +# Note: Add any additional requirements to setup.py's install_requires field --editable . wheel From 2c4d7a2504a08f9ebbcc728b35bcd41bfc74ec63 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 15 Apr 2021 11:53:57 -0400 Subject: [PATCH 223/282] Remove user from CODEOWNERS Due to her departure this removes hillaryj from the default CODEOWNERS we use in our projects. --- .github/CODEOWNERS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9c3d21f..371258c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,8 +3,8 @@ # These owners will be the default owners for everything in the # repo. Unless a later match takes precedence, these owners will be # requested for review when someone opens a pull request. -* @dav3r @felddy @hillaryj @jsf9k @mcdonnnj +* @dav3r @felddy @jsf9k @mcdonnnj # These folks own any files in the .github directory at the root of # the repository and any of its subdirectories. -/.github/ @dav3r @felddy @hillaryj @jsf9k @mcdonnnj +/.github/ @dav3r @felddy @jsf9k @mcdonnnj From 03494999d60d682a882a5191b16a41cdcdf96214 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 15 Apr 2021 11:36:21 -0400 Subject: [PATCH 224/282] Update pre-commit hooks Hooks updated with the `pre-commit autoupdate` command. Note: The `ansible-lint` hook is intentionally being held back on 4.3.7 because of ongoing issues with the 5.x version and how we use Ansible (standalone Galaxy roles). --- .pre-commit-config.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c851317..cf0330d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.4.0 + rev: v4.0.1 hooks: - id: check-case-conflict - id: check-executables-have-shebangs @@ -31,17 +31,17 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.26.0 + rev: v0.27.1 hooks: - id: markdownlint args: - --config=.mdl_config.json - repo: https://github.com/pre-commit/mirrors-prettier - rev: v2.2.1 + rev: v2.3.0 hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint - rev: v1.26.0 + rev: v1.26.1 hooks: - id: yamllint args: @@ -49,7 +49,7 @@ repos: # Shell script hooks - repo: https://github.com/lovesegfault/beautysh - rev: 6.0.1 + rev: v6.1.0 hooks: - id: beautysh args: @@ -68,25 +68,25 @@ repos: args: - --config=.bandit.yml - repo: https://github.com/psf/black - rev: 20.8b1 + rev: 21.5b2 hooks: - id: black - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.4 + rev: 3.9.2 hooks: - id: flake8 additional_dependencies: - flake8-docstrings - repo: https://github.com/PyCQA/isort - rev: 5.7.0 + rev: 5.8.0 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.800 + rev: v0.812 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v2.10.0 + rev: v2.19.1 hooks: - id: pyupgrade @@ -101,7 +101,7 @@ repos: # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.45.0 + rev: v1.50.0 hooks: - id: terraform_fmt # There are ongoing issues with how this command works. This issue From 23b324befc9cdc26ffcf1af6f333b63c545dfb7b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 1 Jun 2021 11:14:31 -0400 Subject: [PATCH 225/282] Update configuration instructions for pyenv With the release of pyenv v2.0.0 there is a breaking change around startup logic that necessitates an update for our pyenv setup instructions. Also add a statement about how to get configuration instructions from pyenv itself. --- CONTRIBUTING.md | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6434d30..27bb9e7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,6 +64,9 @@ installation is as simple as `brew install pyenv pyenv-virtualenv` and adding this to your profile: ```bash +export PYENV_ROOT="$HOME/.pyenv" +export PATH="$PYENV_ROOT/bin:$PATH" +eval "$(pyenv init --path)" eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" ``` @@ -80,14 +83,35 @@ On WSL you should treat your platform as whatever Linux distribution you've chosen to install. Once you have installed `pyenv` you will need to add the following -lines to your `.bashrc`: +lines to your `.bash_profile` (or `.profile`): + +```bash +export PYENV_ROOT="$HOME/.pyenv" +export PATH="$PYENV_ROOT/bin:$PATH" +eval "$(pyenv init --path)" +``` + +and then add the following lines to your `.bashrc`: ```bash -export PATH="$PATH:$HOME/.pyenv/bin" eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" ``` +If you want more information about setting up `pyenv` once installed, please run + +```console +pyenv init +``` + +and + +```console +pyenv virtualenv-init +``` + +for the current configuration instructions. + If you are using a shell other than `bash` you should follow the instructions that the `pyenv-installer` script outputs. From 4baa1ed6a65f58afd667066885403e6189b56d2a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 7 Jun 2021 07:46:48 -0400 Subject: [PATCH 226/282] Standardize the layout of the Lineage configuration file --- .github/lineage.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/lineage.yml b/.github/lineage.yml index 8dfc20b..49f9c4f 100644 --- a/.github/lineage.yml +++ b/.github/lineage.yml @@ -1,6 +1,5 @@ --- -version: "1" - lineage: skeleton: remote-url: https://github.com/cisagov/skeleton-generic.git +version: '1' From 249bbbb49292ef5f4ed85f0792b416e817604b10 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 7 Jun 2021 07:53:22 -0400 Subject: [PATCH 227/282] Add to the cache keys for the GitHub Actions workflow This adds a hash of `setup.py` to the cache keys used in the GitHub Actions workflow for Python projects. --- .github/workflows/build.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 695e8cc..7ca5aeb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,7 +31,8 @@ jobs: key: "${{ env.BASE_CACHE_KEY }}\ ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/requirements.txt') }}-\ - ${{ hashFiles('**/.pre-commit-config.yaml') }}" + ${{ hashFiles('**/.pre-commit-config.yaml') }}-\ + ${{ hashFiles('setup.py') }}" restore-keys: | ${{ env.BASE_CACHE_KEY }} - name: Install dependencies @@ -61,7 +62,8 @@ jobs: path: ${{ env.PIP_CACHE_DIR }} key: "${{ env.BASE_CACHE_KEY }}\ ${{ hashFiles('**/requirements-test.txt') }}-\ - ${{ hashFiles('**/requirements.txt') }}" + ${{ hashFiles('**/requirements.txt') }}-\ + ${{ hashFiles('setup.py') }}" restore-keys: | ${{ env.BASE_CACHE_KEY }} - name: Install dependencies @@ -97,7 +99,8 @@ jobs: path: ${{ env.PIP_CACHE_DIR }} key: "${{ env.BASE_CACHE_KEY }}\ ${{ hashFiles('**/requirements-test.txt') }}-\ - ${{ hashFiles('**/requirements.txt') }}" + ${{ hashFiles('**/requirements.txt') }}-\ + ${{ hashFiles('setup.py') }}" restore-keys: | ${{ env.BASE_CACHE_KEY }} - name: Install dependencies @@ -127,7 +130,8 @@ jobs: with: path: ${{ env.PIP_CACHE_DIR }} key: "${{ env.BASE_CACHE_KEY }}\ - ${{ hashFiles('**/requirements.txt') }}" + ${{ hashFiles('**/requirements.txt') }}-\ + ${{ hashFiles('setup.py') }}" restore-keys: | ${{ env.BASE_CACHE_KEY }} - name: Install dependencies From 449eef422bf7cb8469a25996126fac26ae899fb4 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 7 Jun 2021 16:53:03 -0400 Subject: [PATCH 228/282] Add comments explaining different hashFiles() argument The other cache keys for our GHA jobs are in the format '**/' so that any file with that name is used in the repository. However, for Python packages they may have a 'setup.py' as part of their internal codebase that does not impact environment requirements. As a result we only want to use the 'setup.py' that is in the root of the repository and is used to install the package. Co-authored-by: dav3r --- .github/workflows/build.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7ca5aeb..c054b90 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -28,6 +28,9 @@ jobs: path: | ${{ env.PIP_CACHE_DIR }} ${{ env.PRE_COMMIT_CACHE_DIR }} + # We do not use '**/setup.py' in the cache key so only the 'setup.py' + # file in the root of the repository is used. This is in case a Python + # package were to have a 'setup.py' as part of its internal codebase. key: "${{ env.BASE_CACHE_KEY }}\ ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/requirements.txt') }}-\ @@ -60,6 +63,9 @@ jobs: py${{ steps.setup-python.outputs.python-version }}-" with: path: ${{ env.PIP_CACHE_DIR }} + # We do not use '**/setup.py' in the cache key so only the 'setup.py' + # file in the root of the repository is used. This is in case a Python + # package were to have a 'setup.py' as part of its internal codebase. key: "${{ env.BASE_CACHE_KEY }}\ ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/requirements.txt') }}-\ @@ -97,6 +103,9 @@ jobs: py${{ steps.setup-python.outputs.python-version }}-" with: path: ${{ env.PIP_CACHE_DIR }} + # We do not use '**/setup.py' in the cache key so only the 'setup.py' + # file in the root of the repository is used. This is in case a Python + # package were to have a 'setup.py' as part of its internal codebase. key: "${{ env.BASE_CACHE_KEY }}\ ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/requirements.txt') }}-\ @@ -129,6 +138,9 @@ jobs: py${{ steps.setup-python.outputs.python-version }}-" with: path: ${{ env.PIP_CACHE_DIR }} + # We do not use '**/setup.py' in the cache key so only the 'setup.py' + # file in the root of the repository is used. This is in case a Python + # package were to have a 'setup.py' as part of its internal codebase. key: "${{ env.BASE_CACHE_KEY }}\ ${{ hashFiles('**/requirements.txt') }}-\ ${{ hashFiles('setup.py') }}" From 1e8f8223910f41294c16bfebea332c80fd83573c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 18 Jun 2021 03:33:26 -0400 Subject: [PATCH 229/282] Add style enforcement rules Add rules to enforce ATX-closed headers, dashes for unordered list elements, and `1.` for ordered list elements. --- .mdl_config.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.mdl_config.json b/.mdl_config.json index 7a6f3f8..8266cdb 100644 --- a/.mdl_config.json +++ b/.mdl_config.json @@ -1,4 +1,10 @@ { + "MD003": { + "style": "atx_closed" + }, + "MD004": { + "style": "dash" + }, "MD013": { "code_blocks": false, "tables": false @@ -6,5 +12,8 @@ "MD024": { "allow_different_nesting": true }, + "MD029": { + "style": "one" + }, "default": true } From afc6bd6f6d754f8c2bc5675411535b0c61e0ec78 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 18 Jun 2021 03:37:08 -0400 Subject: [PATCH 230/282] Add rule for image headers Add

and tags to the allowed list for MD033 (HTML elements) to support using an image as the first thing in a markdown file (header image). --- .mdl_config.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.mdl_config.json b/.mdl_config.json index 8266cdb..38bc045 100644 --- a/.mdl_config.json +++ b/.mdl_config.json @@ -15,5 +15,11 @@ "MD029": { "style": "one" }, + "MD033": { + "allowed_elements": [ + "h1", + "img" + ] + }, "default": true } From ce173f401d8eec4f2caf3cf8174a5b53f03222da Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 18 Jun 2021 03:51:38 -0400 Subject: [PATCH 231/282] Switch to a YAML markdownlint configuration file This converts the existing `.mdl_config.json` file to an equivalent `.mdl_config.yaml` file. The reference in the markdownlint pre-commit hook configuration is updated to match. Co-authored-by: Shane Frasier --- .mdl_config.json | 25 ------------------------- .mdl_config.yaml | 24 ++++++++++++++++++++++++ .pre-commit-config.yaml | 2 +- 3 files changed, 25 insertions(+), 26 deletions(-) delete mode 100644 .mdl_config.json create mode 100644 .mdl_config.yaml diff --git a/.mdl_config.json b/.mdl_config.json deleted file mode 100644 index 38bc045..0000000 --- a/.mdl_config.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "MD003": { - "style": "atx_closed" - }, - "MD004": { - "style": "dash" - }, - "MD013": { - "code_blocks": false, - "tables": false - }, - "MD024": { - "allow_different_nesting": true - }, - "MD029": { - "style": "one" - }, - "MD033": { - "allowed_elements": [ - "h1", - "img" - ] - }, - "default": true -} diff --git a/.mdl_config.yaml b/.mdl_config.yaml new file mode 100644 index 0000000..a04720a --- /dev/null +++ b/.mdl_config.yaml @@ -0,0 +1,24 @@ +--- + +default: true + +MD003: + style: "atx_closed" + +MD004: + style: "dash" + +MD013: + code_blocks: false + tables: false + +MD024: + allow_different_nesting: true + +MD029: + style: "one" + +MD033: + allowed_elements: + - h1 + - img diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cf0330d..6b87ab0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,7 +35,7 @@ repos: hooks: - id: markdownlint args: - - --config=.mdl_config.json + - --config=.mdl_config.yaml - repo: https://github.com/pre-commit/mirrors-prettier rev: v2.3.0 hooks: From f2a423095efe2f1a96ce2621352ee7cb4dd0458a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 18 Jun 2021 03:59:36 -0400 Subject: [PATCH 232/282] Add comments to markdownlint configuration Now that this is a YAML file we can add comments explaining the rule modifications we use. This will make it easier to edit or expand in the future. --- .mdl_config.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.mdl_config.yaml b/.mdl_config.yaml index a04720a..a2f08f3 100644 --- a/.mdl_config.yaml +++ b/.mdl_config.yaml @@ -1,24 +1,40 @@ --- +# Default state for all rules default: true +# MD003/heading-style/header-style - Heading style MD003: + # Enforce the ATX-closed style of header style: "atx_closed" +# MD004/ul-style - Unordered list style MD004: + # Enforce dashes for unordered lists style: "dash" +# MD013/line-length - Line length MD013: + # Do not enforce for code blocks code_blocks: false + # Do not enforce for tables tables: false +# MD024/no-duplicate-heading/no-duplicate-header - Multiple headings with the +# same content MD024: + # Allow headers with the same content as long as they are not in the same + # parent heading allow_different_nesting: true +# MD029/ol-prefix - Ordered list item prefix MD029: + # Enforce the `1.` style for ordered lists style: "one" +# MD033/no-inline-html - Inline HTML MD033: + # The h1 and img elements are allowed to permit header images allowed_elements: - h1 - img From d4781ee177698490b740c5dc0e3ae90282c70618 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 24 Jun 2021 01:38:44 -0400 Subject: [PATCH 233/282] Add the validate_manifest hook from pre-commit This hook will validate any pre-commit hook manifest files in the repository. --- .pre-commit-config.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cf0330d..82a5e46 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -47,6 +47,12 @@ repos: args: - --strict + # pre-commit hooks + - repo: https://github.com/pre-commit/pre-commit + rev: v2.13.0 + hooks: + - id: validate_manifest + # Shell script hooks - repo: https://github.com/lovesegfault/beautysh rev: v6.1.0 From 7ae8bea9125fbf458c0fb824c7d64a6f33c6b2c2 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 25 Jun 2021 09:09:16 -0400 Subject: [PATCH 234/282] Add assertion to verify that the root logger's logging level is set correctly See the discussion here for more context: https://github.com/cisagov/pe-reports/pull/6#discussion_r657329612 --- tests/test_example.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_example.py b/tests/test_example.py index 3a22848..fe08eb7 100644 --- a/tests/test_example.py +++ b/tests/test_example.py @@ -83,6 +83,9 @@ def test_log_levels(level): assert ( logging.root.hasHandlers() is True ), "root logger should now have a handler" + assert ( + logging.getLevelName(logging.root.getEffectiveLevel()) == level.upper() + ), f"root logger level should be set to {level.upper()}" assert return_code == 0, "main() should return success (0)" From 106af21c04ae34d0402b9cfc59f386e2756776bd Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 9 Jul 2021 13:34:16 -0400 Subject: [PATCH 235/282] Install terraform and packer for the linting job We should be doing this because the Packer and Terraform pre-commit hooks leverage the corresponding executables; therefore, it makes sense to go ahead and install the particular versions of those executables that we support. Also add support for optionally debugging via tmate. See also cisagov/skeleton-generic#74. --- .github/workflows/build.yml | 63 ++++++++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5c65f71..8fa1b2f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,13 +8,16 @@ on: types: [apb] env: + CURL_CACHE_DIR: ~/.cache/curl PIP_CACHE_DIR: ~/.cache/pip PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit + RUN_TMATE: ${{ secrets.RUN_TMATE }} jobs: lint: runs-on: ubuntu-latest steps: + - uses: cisagov/setup-env-github-action@develop - uses: actions/checkout@v2 - id: setup-python uses: actions/setup-python@v2 @@ -23,17 +26,72 @@ jobs: - uses: actions/cache@v2 env: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ - py${{ steps.setup-python.outputs.python-version }}-" + py${{ steps.setup-python.outputs.python-version }}-\ + go${{ env.GO_VERSION }}-\ + packer${{ env.PACKER_VERSION }}-\ + tf${{ env.TERRAFORM_VERSION }}-" with: + # Note that the .terraform directory IS NOT included in the + # cache because if we were caching, then we would need to use + # the `-upgrade=true` option. This option blindly pulls down the + # latest modules and providers instead of checking to see if an + # update is required. That behavior defeats the benefits of caching. + # so there is no point in doing it for the .terraform directory. path: | ${{ env.PIP_CACHE_DIR }} ${{ env.PRE_COMMIT_CACHE_DIR }} + ${{ env.CURL_CACHE_DIR }} + ${{ steps.go-cache.outputs.dir }} key: "${{ env.BASE_CACHE_KEY }}\ ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/requirements.txt') }}-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" restore-keys: | ${{ env.BASE_CACHE_KEY }} + - uses: actions/setup-go@v2 + with: + go-version: '1.16' + - name: Store installed Go version + run: | + echo "GO_VERSION="\ + "$(go version | sed 's/^go version go\([0-9.]\+\) .*/\1/')" \ + >> $GITHUB_ENV + - name: Lookup go cache directory + id: go-cache + run: | + echo "::set-output name=dir::$(go env GOCACHE)" + - name: Install Packer + run: | + mkdir -p ${{ env.CURL_CACHE_DIR }} + PACKER_ZIP="packer_${PACKER_VERSION}_linux_amd64.zip" + curl --output ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ + --time-cond ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ + --location \ + "https://releases.hashicorp.com/packer/${PACKER_VERSION}/${PACKER_ZIP}" + sudo unzip -o -d /usr/local/bin \ + ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" + - name: Install Terraform + run: | + mkdir -p ${{ env.CURL_CACHE_DIR }} + TERRAFORM_ZIP="terraform_${TERRAFORM_VERSION}_linux_amd64.zip" + curl --output ${{ env.CURL_CACHE_DIR }}/"${TERRAFORM_ZIP}" \ + --time-cond ${{ env.CURL_CACHE_DIR }}/"${TERRAFORM_ZIP}" \ + --location \ + "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/${TERRAFORM_ZIP}" + sudo unzip -d /opt/terraform \ + ${{ env.CURL_CACHE_DIR }}/"${TERRAFORM_ZIP}" + sudo ln -s /opt/terraform/terraform /usr/bin/terraform + sudo mv /usr/local/bin/terraform /usr/local/bin/terraform-default + sudo ln -s /opt/terraform/terraform /usr/local/bin/terraform + - name: Install Terraform-docs + run: GO111MODULE=on go get github.com/terraform-docs/terraform-docs + - name: Find and initialize Terraform directories + run: | + for path in $(find . -not \( -type d -name ".terraform" -prune \) \ + -type f -iname "*.tf" -exec dirname "{}" \; | sort -u); do \ + echo "Initializing '$path'..."; \ + terraform init -input=false -backend=false "$path"; \ + done - name: Install dependencies run: | python -m pip install --upgrade pip @@ -42,3 +100,6 @@ jobs: run: pre-commit install-hooks - name: Run pre-commit on all files run: pre-commit run --all-files + - name: Setup tmate debug session + uses: mxschmitt/action-tmate@v3 + if: env.RUN_TMATE From c4810439814c3ebed6dd05f3b690b460670cd878 Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Fri, 9 Jul 2021 22:43:09 -0400 Subject: [PATCH 236/282] Break out the curl cache creation into its own step Co-authored-by: Nick M. <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8fa1b2f..bc632c3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -60,9 +60,10 @@ jobs: id: go-cache run: | echo "::set-output name=dir::$(go env GOCACHE)" + - name: Setup curl cache + run: mkdir -p ${{ env.CURL_CACHE_DIR }} - name: Install Packer run: | - mkdir -p ${{ env.CURL_CACHE_DIR }} PACKER_ZIP="packer_${PACKER_VERSION}_linux_amd64.zip" curl --output ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ --time-cond ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ @@ -72,7 +73,6 @@ jobs: ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" - name: Install Terraform run: | - mkdir -p ${{ env.CURL_CACHE_DIR }} TERRAFORM_ZIP="terraform_${TERRAFORM_VERSION}_linux_amd64.zip" curl --output ${{ env.CURL_CACHE_DIR }}/"${TERRAFORM_ZIP}" \ --time-cond ${{ env.CURL_CACHE_DIR }}/"${TERRAFORM_ZIP}" \ From 70414cff28c661c3b76425edf5021f213f505413 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 9 Jul 2021 22:46:47 -0400 Subject: [PATCH 237/282] Remove unnecessary line in tasks There is no reason to create /usr/bin/terraform. This is a vestige of an earlier age. Co-authored-by: Nick M. <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bc632c3..3946d90 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -80,7 +80,6 @@ jobs: "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/${TERRAFORM_ZIP}" sudo unzip -d /opt/terraform \ ${{ env.CURL_CACHE_DIR }}/"${TERRAFORM_ZIP}" - sudo ln -s /opt/terraform/terraform /usr/bin/terraform sudo mv /usr/local/bin/terraform /usr/local/bin/terraform-default sudo ln -s /opt/terraform/terraform /usr/local/bin/terraform - name: Install Terraform-docs From b629f7f623490217fbd43d76fd77b4638cd4a4ec Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 9 Jul 2021 22:48:21 -0400 Subject: [PATCH 238/282] Modify the Packer installation to model that of Terraform The Terraform installation does not destroy the existing system Terraform installation, and neither should the Packer installation. Co-authored-by: Nick M. <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3946d90..871bee7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -69,8 +69,10 @@ jobs: --time-cond ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ --location \ "https://releases.hashicorp.com/packer/${PACKER_VERSION}/${PACKER_ZIP}" - sudo unzip -o -d /usr/local/bin \ + sudo unzip -d /opt/packer \ ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" + sudo mv /usr/local/bin/packer /usr/local/bin/packer-default + sudo ln -s /opt/packer/packer /usr/local/bin/packer - name: Install Terraform run: | TERRAFORM_ZIP="terraform_${TERRAFORM_VERSION}_linux_amd64.zip" From 181d1b2fafa211fb7cae5b6023e1b5271b59bbda Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Sat, 10 Jul 2021 22:36:45 -0400 Subject: [PATCH 239/282] Install a specific version of terraform-docs Note that this change is dependent on the merging of cisagov/setup-env-github-action#31. Co-authored-by: Nick M. <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 871bee7..c8a1426 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -85,7 +85,9 @@ jobs: sudo mv /usr/local/bin/terraform /usr/local/bin/terraform-default sudo ln -s /opt/terraform/terraform /usr/local/bin/terraform - name: Install Terraform-docs - run: GO111MODULE=on go get github.com/terraform-docs/terraform-docs + run: | + GO111MODULE=on go get \ + github.com/terraform-docs/terraform-docs@${TERRAFORM_DOCS_VERSION} - name: Find and initialize Terraform directories run: | for path in $(find . -not \( -type d -name ".terraform" -prune \) \ From bb6e566e3a8e1069ca2c6a1f441f67fc4c176685 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Sun, 11 Jul 2021 21:59:58 -0400 Subject: [PATCH 240/282] Move go installation so that it takes place before the cache task Some variables defined in the go installation are used in the cache task, so the go installation must happen first. Co-authored-by: Nick M. <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c8a1426..04159c2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,6 +23,20 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.9 + # GO_VERSION and GOCACHE are used by the cache task, so the go + # installation must happen before that. + - uses: actions/setup-go@v2 + with: + go-version: '1.16' + - name: Store installed Go version + run: | + echo "GO_VERSION="\ + "$(go version | sed 's/^go version go\([0-9.]\+\) .*/\1/')" \ + >> $GITHUB_ENV + - name: Lookup go cache directory + id: go-cache + run: | + echo "::set-output name=dir::$(go env GOCACHE)" - uses: actions/cache@v2 env: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ @@ -48,18 +62,6 @@ jobs: ${{ hashFiles('**/.pre-commit-config.yaml') }}" restore-keys: | ${{ env.BASE_CACHE_KEY }} - - uses: actions/setup-go@v2 - with: - go-version: '1.16' - - name: Store installed Go version - run: | - echo "GO_VERSION="\ - "$(go version | sed 's/^go version go\([0-9.]\+\) .*/\1/')" \ - >> $GITHUB_ENV - - name: Lookup go cache directory - id: go-cache - run: | - echo "::set-output name=dir::$(go env GOCACHE)" - name: Setup curl cache run: mkdir -p ${{ env.CURL_CACHE_DIR }} - name: Install Packer From 337d1efb8f72c11cae6b83f3f5e63e8187599470 Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Mon, 12 Jul 2021 09:06:41 -0400 Subject: [PATCH 241/282] Capitalize Go for consistency Co-authored-by: Nick M. <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 04159c2..d84b7da 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,7 +23,7 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.9 - # GO_VERSION and GOCACHE are used by the cache task, so the go + # GO_VERSION and GOCACHE are used by the cache task, so the Go # installation must happen before that. - uses: actions/setup-go@v2 with: @@ -33,7 +33,7 @@ jobs: echo "GO_VERSION="\ "$(go version | sed 's/^go version go\([0-9.]\+\) .*/\1/')" \ >> $GITHUB_ENV - - name: Lookup go cache directory + - name: Lookup Go cache directory id: go-cache run: | echo "::set-output name=dir::$(go env GOCACHE)" From 8ee2116f428f1738540f0cdf261f1e4f1c15d092 Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Tue, 13 Jul 2021 17:02:13 -0400 Subject: [PATCH 242/282] Prefer the newer "go install" syntax As of [Go 1.16](https://tip.golang.org/doc/go1.16#go-command) the `GO111MODULE` environment variable defaults to `on` and `go get` has been deprecated for module installation. Co-authored-by: Nick M. <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d84b7da..73f345a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -88,7 +88,7 @@ jobs: sudo ln -s /opt/terraform/terraform /usr/local/bin/terraform - name: Install Terraform-docs run: | - GO111MODULE=on go get \ + go install \ github.com/terraform-docs/terraform-docs@${TERRAFORM_DOCS_VERSION} - name: Find and initialize Terraform directories run: | From e2a729d0b11ab74207a3bb77367d8e9d8c577889 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 13 Jul 2021 17:42:36 -0400 Subject: [PATCH 243/282] Install the shfmt tool for GHA The `shfmt` tool does not ship on the GitHub Actions runners so we must install it manually. --- .github/workflows/build.yml | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5c65f71..9dd5f7a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,25 +15,44 @@ jobs: lint: runs-on: ubuntu-latest steps: + - uses: cisagov/setup-env-github-action@develop - uses: actions/checkout@v2 - id: setup-python uses: actions/setup-python@v2 with: python-version: 3.9 + # GO_VERSION and GOCACHE are used by the cache task, so the Go + # installation must happen before that. + - uses: actions/setup-go@v2 + with: + go-version: '1.16' + - name: Store installed Go version + run: | + echo "GO_VERSION="\ + "$(go version | sed 's/^go version go\([0-9.]\+\) .*/\1/')" \ + >> $GITHUB_ENV + - name: Lookup Go cache directory + id: go-cache + run: | + echo "::set-output name=dir::$(go env GOCACHE)" - uses: actions/cache@v2 env: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ - py${{ steps.setup-python.outputs.python-version }}-" + py${{ steps.setup-python.outputs.python-version }}-\ + go${{ env.GO_VERSION }}-" with: path: | ${{ env.PIP_CACHE_DIR }} ${{ env.PRE_COMMIT_CACHE_DIR }} + ${{ steps.go-cache.outputs.dir }} key: "${{ env.BASE_CACHE_KEY }}\ ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/requirements.txt') }}-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" restore-keys: | ${{ env.BASE_CACHE_KEY }} + - name: Install shfmt + run: go install mvdan.cc/sh/v3/cmd/shfmt@${SHFMT_VERSION} - name: Install dependencies run: | python -m pip install --upgrade pip From 406b6880bd25a8592ad235102d4e832e05ab38e3 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 13 Jul 2021 17:53:55 -0400 Subject: [PATCH 244/282] Replace the beautysh hook with pre-commit-shfmt We have had a difficult time with how beautysh parses some shellscripts. I went in pursuit of an alternative and I believe shfmt to be a good alternative. Co-authored-by: Shane Frasier --- .pre-commit-config.yaml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cf0330d..c915aa9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,13 +48,20 @@ repos: - --strict # Shell script hooks - - repo: https://github.com/lovesegfault/beautysh - rev: v6.1.0 + - repo: https://github.com/cisagov/pre-commit-shfmt + rev: v0.0.2 hooks: - - id: beautysh + - id: shfmt args: - - --indent-size + # Indent by two spaces + - -i - '2' + # Binary operators may start a line + - -bn + # Switch cases are indented + - -ci + # Redirect operators are followed by a space + - -sr - repo: https://github.com/detailyang/pre-commit-shell rev: 1.0.5 hooks: From 2b48e75b23cb80af9e97098da2dd6b9fb5eea2e4 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 13 Jul 2021 18:08:51 -0400 Subject: [PATCH 245/282] Apply changes from the shfmt pre-commit hook --- setup-env | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/setup-env b/setup-env index 1579e04..5d7f673 100755 --- a/setup-env +++ b/setup-env @@ -4,7 +4,8 @@ set -o nounset set -o errexit set -o pipefail -USAGE=$(cat << 'END_OF_LINE' +USAGE=$( + cat << 'END_OF_LINE' Configure a developement environment for this repository. It does the following: @@ -35,17 +36,17 @@ FORCE=0 PARAMS="" # Parse command line arguments -while (( "$#" )); do +while (("$#")); do case "$1" in - -f|--force) + -f | --force) FORCE=1 shift ;; - -h|--help) + -h | --help) echo "${USAGE}" exit 0 ;; - -i|--install-hooks) + -i | --install-hooks) INSTALL_HOOKS=1 shift ;; @@ -160,7 +161,8 @@ pre-commit install ${INSTALL_HOOKS:+"--install-hooks"} # This could fail if the remotes are already setup, but that is ok. set +o errexit -eval "$(python3 << 'END_OF_LINE' +eval "$( + python3 << 'END_OF_LINE' from pathlib import Path import yaml import sys From 1df17570c764a6765e7b699bcbbc09ca42fcabc9 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 14 Jul 2021 18:23:59 -0400 Subject: [PATCH 246/282] Remove unnecessary boilerplate This package has a `__main__.py` file which removes the need to include this boilerplate. --- src/example/example.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/example/example.py b/src/example/example.py index 73faa33..a0dcb68 100644 --- a/src/example/example.py +++ b/src/example/example.py @@ -102,7 +102,3 @@ def main() -> int: # Stop logging and clean up logging.shutdown() return 0 - - -if __name__ == "__main__": - sys.exit(main()) From ad5009c1956f476cf9968e1d28bb7da64cfb289b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 14 Jul 2021 18:29:45 -0400 Subject: [PATCH 247/282] Explicitly call sys.exit for errors Instead of returning a value the main loop should explicitly call `sys.exit()` if an error is encountered that would result in a non-zero exit code. --- src/example/example.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/example/example.py b/src/example/example.py index a0dcb68..28b304e 100644 --- a/src/example/example.py +++ b/src/example/example.py @@ -45,7 +45,7 @@ def example_div(dividend: float, divisor: float) -> float: return dividend / divisor -def main() -> int: +def main() -> None: """Set up logging and call the example function.""" args: Dict[str, str] = docopt.docopt(__doc__, version=__version__) # Validate and convert arguments as needed @@ -73,7 +73,7 @@ def main() -> int: except SchemaError as err: # Exit because one or more of the arguments were invalid print(err, file=sys.stderr) - return 1 + sys.exit(1) # Assign validated arguments to variables dividend: int = validated_args[""] @@ -101,4 +101,3 @@ def main() -> int: # Stop logging and clean up logging.shutdown() - return 0 From 8a8785d16f5cb43b8e55141e7b350e7b0d79d844 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 14 Jul 2021 18:31:49 -0400 Subject: [PATCH 248/282] Update function type hints The type hints for the example_div function indicate it takes two float values, but the command line interface indicates (and checks) that it takes integer values. --- src/example/example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/example/example.py b/src/example/example.py index 28b304e..9267cc5 100644 --- a/src/example/example.py +++ b/src/example/example.py @@ -35,7 +35,7 @@ DEFAULT_ECHO_MESSAGE: str = "Hello World from the example default!" -def example_div(dividend: float, divisor: float) -> float: +def example_div(dividend: int, divisor: int) -> float: """Print some logging messages.""" logging.debug("This is a debug message") logging.info("This is an info message") From 29ade6fa6ea6187d4059a7c6a8ef630458337c1e Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 14 Jul 2021 18:36:43 -0400 Subject: [PATCH 249/282] Do not use f-strings in logging messages C-style '%' formatting should be used to any calls to the `logging` library. --- src/example/example.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/example/example.py b/src/example/example.py index 9267cc5..d3eda19 100644 --- a/src/example/example.py +++ b/src/example/example.py @@ -85,11 +85,11 @@ def main() -> None: format="%(asctime)-15s %(levelname)s %(message)s", level=log_level.upper() ) - logging.info(f"{dividend} / {divisor} == {example_div(dividend, divisor)}") + logging.info("%d / %d == %f", dividend, divisor, example_div(dividend, divisor)) # Access some data from an environment variable message: str = os.getenv("ECHO_MESSAGE", DEFAULT_ECHO_MESSAGE) - logging.info(f'ECHO_MESSAGE="{message}"') + logging.info('ECHO_MESSAGE="%s"', message) # Access some data from our package data (see the setup.py) secret_message: str = ( @@ -97,7 +97,7 @@ def main() -> None: .decode("utf-8") .strip() ) - logging.info(f'Secret="{secret_message}"') + logging.info('Secret="%s"', secret_message) # Stop logging and clean up logging.shutdown() From 015d47e7e9d4d4570bb42396e5a94495edb5b19d Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 14 Jul 2021 18:45:04 -0400 Subject: [PATCH 250/282] Update testing for exit code changes The testing needs to be updated now that `example.main()` only returns a `None` and error exits are done by directly calling `sys.exit()`. --- tests/test_example.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/tests/test_example.py b/tests/test_example.py index 3a22848..5ea4c5c 100644 --- a/tests/test_example.py +++ b/tests/test_example.py @@ -79,18 +79,26 @@ def test_log_levels(level): assert ( logging.root.hasHandlers() is False ), "root logger should not have handlers yet" - return_code = example.example.main() + return_code = None + try: + example.example.main() + except SystemExit as sys_exit: + return_code = sys_exit.code + assert return_code is None, "main() should return success" assert ( logging.root.hasHandlers() is True ), "root logger should now have a handler" - assert return_code == 0, "main() should return success (0)" def test_bad_log_level(): """Validate bad log-level argument returns error.""" with patch.object(sys, "argv", ["bogus", "--log-level=emergency", "1", "1"]): - return_code = example.example.main() - assert return_code == 1, "main() should return failure" + return_code = None + try: + example.example.main() + except SystemExit as sys_exit: + return_code = sys_exit.code + assert return_code == 1, "main() should exit with error" @pytest.mark.parametrize("dividend, divisor, quotient", div_params) @@ -124,5 +132,9 @@ def test_zero_division(): def test_zero_divisor_argument(): """Verify that a divisor of zero is handled as expected.""" with patch.object(sys, "argv", ["bogus", "1", "0"]): - return_code = example.example.main() + return_code = None + try: + example.example.main() + except SystemExit as sys_exit: + return_code = sys_exit.code assert return_code == 1, "main() should exit with error" From 1708b5c45ac12f9acfb447e532c4fc5746a80d23 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 13 Jul 2021 11:27:26 -0400 Subject: [PATCH 251/282] Update pre-commit hooks This is performed by running `pre-commit autoupdate`, but with the `ansible-lint` hook held back manually. --- .pre-commit-config.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cf0330d..cf319bf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,7 +37,7 @@ repos: args: - --config=.mdl_config.json - repo: https://github.com/pre-commit/mirrors-prettier - rev: v2.3.0 + rev: v2.3.2 hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint @@ -68,7 +68,7 @@ repos: args: - --config=.bandit.yml - repo: https://github.com/psf/black - rev: 21.5b2 + rev: 21.7b0 hooks: - id: black - repo: https://gitlab.com/pycqa/flake8 @@ -78,15 +78,15 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/PyCQA/isort - rev: 5.8.0 + rev: 5.9.2 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.812 + rev: v0.910 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v2.19.1 + rev: v2.21.2 hooks: - id: pyupgrade @@ -123,7 +123,7 @@ repos: # Docker hooks - repo: https://github.com/IamTheFij/docker-pre-commit - rev: v2.0.0 + rev: v2.0.1 hooks: - id: docker-compose-check From 63cf76fd694d13e431f820835a8fb0a4a993a3f9 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 20 Jul 2021 12:55:26 -0400 Subject: [PATCH 252/282] Changes from the pre-commit-shfmt hook --- bump_version.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bump_version.sh b/bump_version.sh index c0bf25c..e1324b8 100755 --- a/bump_version.sh +++ b/bump_version.sh @@ -12,12 +12,11 @@ HELP_INFORMATION="bump_version.sh (show|major|minor|patch|prerelease|build|final old_version=$(sed -n "s/^__version__ = \"\(.*\)\"$/\1/p" $VERSION_FILE) -if [ $# -ne 1 ] -then +if [ $# -ne 1 ]; then echo "$HELP_INFORMATION" else case $1 in - major|minor|patch|prerelease|build) + major | minor | patch | prerelease | build) new_version=$(python -c "import semver; print(semver.bump_$1('$old_version'))") echo Changing version from "$old_version" to "$new_version" # A temp file is used to provide compatability with macOS development From 77c19cb1c5a43e0f100f04caaaa913e8baaab9de Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 20 Jul 2021 13:04:28 -0400 Subject: [PATCH 253/282] Add stubs required by mypy pre-commit hook With the release of 0.900 mypy moved to a modular stub model. This means that only stdlib stubs are "baked in" and everything else must be installed separately. Since the mypy hook runs in a venv the stubs it recommends need to be installed as additional dependencies. --- .pre-commit-config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 74f3836..db9f507 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -108,6 +108,8 @@ repos: rev: v0.910 hooks: - id: mypy + additional_dependencies: + - types-setuptools - repo: https://github.com/asottile/pyupgrade rev: v2.21.2 hooks: From 3452b9de398c48d73f06d2fecc666d07341a9038 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 20 Jul 2021 15:14:19 -0400 Subject: [PATCH 254/282] Install mypy type stubs as part of setup-env We already do this as part of the mypy pre-commit hook, but this way mypy is also ready to run manually in the developer's local Python virtual env. --- setup-env | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 5d7f673..6dd2079 100755 --- a/setup-env +++ b/setup-env @@ -12,7 +12,7 @@ It does the following: - Verifies pyenv and pyenv-virtualenv are installed. - Creates a Python virtual environment. - Configures the activation of the virtual enviroment for the repo directory. - - Installs the requirements needed for development. + - Installs the requirements needed for development (including mypy type stubs). - Installs git pre-commit hooks. - Configures git upstream remote "lineage" repositories. @@ -154,6 +154,9 @@ for req_file in "requirements-dev.txt" "requirements-test.txt" "requirements.txt fi done +# Install all necessary mypy type stubs +mypy --install-types src/ + # Install git pre-commit hooks now or later. pre-commit install ${INSTALL_HOOKS:+"--install-hooks"} From c6de0a188d71008ed490f90e927c6dd5ea468b24 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 29 Jul 2021 15:59:24 -0400 Subject: [PATCH 255/282] Switch to block-style sequence in build workflow Switch from a flow style to a block style sequence in the build workflow. This will improve maintainability by making version changes impact one Python version at a time. --- .github/workflows/build.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7d8c445..ab07eb3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -118,7 +118,11 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: + - 3.6 + - 3.7 + - 3.8 + - 3.9 steps: - uses: actions/checkout@v2 - id: setup-python @@ -199,7 +203,11 @@ jobs: needs: [lint, test] strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: + - 3.6 + - 3.7 + - 3.8 + - 3.9 steps: - uses: actions/checkout@v2 - id: setup-python From a0f24937d650459b8f70fe38f93fcb54fc6ac3d5 Mon Sep 17 00:00:00 2001 From: dav3r Date: Wed, 4 Aug 2021 09:30:18 -0400 Subject: [PATCH 256/282] Fix a typo --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 5d7f673..f526cdb 100755 --- a/setup-env +++ b/setup-env @@ -6,7 +6,7 @@ set -o pipefail USAGE=$( cat << 'END_OF_LINE' -Configure a developement environment for this repository. +Configure a development environment for this repository. It does the following: - Verifies pyenv and pyenv-virtualenv are installed. From f7140d8116532d5d680e10109912fd2741c353ac Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 9 Aug 2021 23:42:42 -0400 Subject: [PATCH 257/282] Use the hashicorp/setup-terraform Action Instead of manually installing a Terraform binary we can leverage the Action provided by Hashicorp to do the same thing. --- .github/workflows/build.yml | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1160e61..6aa2aeb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -75,17 +75,9 @@ jobs: ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" sudo mv /usr/local/bin/packer /usr/local/bin/packer-default sudo ln -s /opt/packer/packer /usr/local/bin/packer - - name: Install Terraform - run: | - TERRAFORM_ZIP="terraform_${TERRAFORM_VERSION}_linux_amd64.zip" - curl --output ${{ env.CURL_CACHE_DIR }}/"${TERRAFORM_ZIP}" \ - --time-cond ${{ env.CURL_CACHE_DIR }}/"${TERRAFORM_ZIP}" \ - --location \ - "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/${TERRAFORM_ZIP}" - sudo unzip -d /opt/terraform \ - ${{ env.CURL_CACHE_DIR }}/"${TERRAFORM_ZIP}" - sudo mv /usr/local/bin/terraform /usr/local/bin/terraform-default - sudo ln -s /opt/terraform/terraform /usr/local/bin/terraform + - uses: hashicorp/setup-terraform@v1 + with: + terraform_version: ${{ env.TERRAFORM_VERSION }} - name: Install shfmt run: go install mvdan.cc/sh/v3/cmd/shfmt@${SHFMT_VERSION} - name: Install Terraform-docs From 7f324b9807549b6b06acd3a43b3636743322e826 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 10 Aug 2021 11:36:33 -0400 Subject: [PATCH 258/282] Add a markdownlint rule for horizontal rules This sets a specific rule for MD035 (Horizontal rule style) instead of the default value of "consistent". --- .mdl_config.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.mdl_config.yaml b/.mdl_config.yaml index a2f08f3..f40ea77 100644 --- a/.mdl_config.yaml +++ b/.mdl_config.yaml @@ -38,3 +38,8 @@ MD033: allowed_elements: - h1 - img + +# MD035/hr-style - Horizontal rule style +MD035: + # Enforce dashes for horizontal rules + style: "---" From 9848ff69448b32b45e31013a4f8709963849659e Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 10 Aug 2021 11:38:39 -0400 Subject: [PATCH 259/282] Add a markdownlint rule for code blocks This sets a specific rule for MD046 (code block style) instead of the default value of "consistent". --- .mdl_config.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.mdl_config.yaml b/.mdl_config.yaml index f40ea77..b36f943 100644 --- a/.mdl_config.yaml +++ b/.mdl_config.yaml @@ -43,3 +43,8 @@ MD033: MD035: # Enforce dashes for horizontal rules style: "---" + +# MD046/code-block-style Code block style +MD046: + # Enforce the fenced style for code blocks + style: "fenced" From 23f9cfd015d5b0de488823130ff0eee4bef2da67 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 9 Aug 2021 23:46:40 -0400 Subject: [PATCH 260/282] Update pre-commit hooks Update pre-commit hooks using `pre-commit autoupdate`. The `ansible-lint` hook is intentionally held back due to issues with switching to v5. --- .pre-commit-config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8ed26ad..5f06453 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.27.1 + rev: v0.28.1 hooks: - id: markdownlint args: @@ -41,7 +41,7 @@ repos: hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint - rev: v1.26.1 + rev: v1.26.2 hooks: - id: yamllint args: @@ -49,7 +49,7 @@ repos: # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v2.13.0 + rev: v2.14.0 hooks: - id: validate_manifest @@ -91,7 +91,7 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/PyCQA/isort - rev: 5.9.2 + rev: 5.9.3 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy @@ -99,7 +99,7 @@ repos: hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v2.21.2 + rev: v2.23.3 hooks: - id: pyupgrade From 60b612ded599d0a24b99c9aa24e33a1f7ba8f598 Mon Sep 17 00:00:00 2001 From: David Redmin Date: Thu, 26 Aug 2021 09:56:34 -0400 Subject: [PATCH 261/282] Enable terraform validate pre-commit hook This hook should now work as expected in most, if not all of our repos now that we are finally updating to terraform 0.13.x (on our way to 1.0.x). --- .pre-commit-config.yaml | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5f06453..da27f36 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -117,22 +117,7 @@ repos: rev: v1.50.0 hooks: - id: terraform_fmt - # There are ongoing issues with how this command works. This issue - # documents the core issue: - # https://github.com/hashicorp/terraform/issues/21408 - # We have seen issues primarily with proxy providers and Terraform code - # that uses remote state. The PR - # https://github.com/hashicorp/terraform/pull/24887 - # has been approved and is part of the 0.13 release to resolve the issue - # with remote states. - # The PR - # https://github.com/hashicorp/terraform/pull/24896 - # is a proprosed fix to deal with `terraform validate` with proxy - # providers (among other configurations). - # We have decided to disable the terraform_validate hook until the issues - # above have been resolved, which we hope will be with the release of - # Terraform 0.13. - # - id: terraform_validate + - id: terraform_validate # Docker hooks - repo: https://github.com/IamTheFij/docker-pre-commit From 6a7fbf07bd371d0493c523ce24647e5c04c77c03 Mon Sep 17 00:00:00 2001 From: David Redmin Date: Thu, 26 Aug 2021 15:33:59 -0400 Subject: [PATCH 262/282] Temporarily use cisagov/setup-env-github-action@improvement/support_tf_0.13 This change will be reverted when testing is completed. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6aa2aeb..b2b73fd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,7 +17,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: cisagov/setup-env-github-action@develop + - uses: cisagov/setup-env-github-action@improvement/support_tf_0.13 - uses: actions/checkout@v2 - id: setup-python uses: actions/setup-python@v2 From 895a692ad8dfc3df50b1c2c9a79bca7166b5f5f2 Mon Sep 17 00:00:00 2001 From: David Redmin Date: Fri, 27 Aug 2021 13:02:00 -0400 Subject: [PATCH 263/282] Remove lint job step to initialize Terraform directories Initialization will now be done during the "terraform validate" step. --- .github/workflows/build.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b2b73fd..f276a36 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -84,13 +84,6 @@ jobs: run: | go install \ github.com/terraform-docs/terraform-docs@${TERRAFORM_DOCS_VERSION} - - name: Find and initialize Terraform directories - run: | - for path in $(find . -not \( -type d -name ".terraform" -prune \) \ - -type f -iname "*.tf" -exec dirname "{}" \; | sort -u); do \ - echo "Initializing '$path'..."; \ - terraform init -input=false -backend=false "$path"; \ - done - name: Install dependencies run: | python -m pip install --upgrade pip From b51dbb577e02baff361a6494e22f61aa517e28d4 Mon Sep 17 00:00:00 2001 From: David Redmin Date: Fri, 27 Aug 2021 14:59:29 -0400 Subject: [PATCH 264/282] Revert "Temporarily use cisagov/setup-env-github-action@improvement/support_tf_0.13" This reverts commit 6a7fbf07bd371d0493c523ce24647e5c04c77c03. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f276a36..090f039 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,7 +17,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: cisagov/setup-env-github-action@improvement/support_tf_0.13 + - uses: cisagov/setup-env-github-action@develop - uses: actions/checkout@v2 - id: setup-python uses: actions/setup-python@v2 From de7e2d437c4d2bd34a1df4f253ad3cd55bacecb6 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 6 Oct 2021 13:28:59 -0400 Subject: [PATCH 265/282] Add Python 3.10 to GHA version matrices Add Python 3.10 to the matrices of versions used by the `test` and `build` steps. --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6cc2c5c..24e9b28 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -115,6 +115,7 @@ jobs: - 3.7 - 3.8 - 3.9 + - 3.10 steps: - uses: actions/checkout@v2 - id: setup-python @@ -200,6 +201,7 @@ jobs: - 3.7 - 3.8 - 3.9 + - 3.10 steps: - uses: actions/checkout@v2 - id: setup-python From c8fa763aee64d6016d06a700f28bf8f67cfb8fa7 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 6 Oct 2021 13:43:41 -0400 Subject: [PATCH 266/282] Force strings for GHA version matrices Ensure that the versions listed in the version matrices used by the `test` and `build` jobs are interpreted as strings. Without this change the version `3.10` is interpreted as `3.1` when converted to a fixnum. Since these are intended to be interpreted as string values we should enforce it for all versions listed. --- .github/workflows/build.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 24e9b28..a2ec81a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -111,11 +111,11 @@ jobs: strategy: matrix: python-version: - - 3.6 - - 3.7 - - 3.8 - - 3.9 - - 3.10 + - "3.6" + - "3.7" + - "3.8" + - "3.9" + - "3.10" steps: - uses: actions/checkout@v2 - id: setup-python @@ -197,11 +197,11 @@ jobs: strategy: matrix: python-version: - - 3.6 - - 3.7 - - 3.8 - - 3.9 - - 3.10 + - "3.6" + - "3.7" + - "3.8" + - "3.9" + - "3.10" steps: - uses: actions/checkout@v2 - id: setup-python From b66988de8c7547e456d570f718244b54a39e79e8 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 29 Sep 2021 13:34:39 -0400 Subject: [PATCH 267/282] Update pre-commit hooks Update pre-commit hooks using `pre-commit autoupdate`. The `ansible-lint` hook is intentionally held back due to issues with switching to v5. --- .pre-commit-config.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index da27f36..3171404 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,17 +31,17 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.28.1 + rev: v0.29.0 hooks: - id: markdownlint args: - --config=.mdl_config.yaml - repo: https://github.com/pre-commit/mirrors-prettier - rev: v2.3.2 + rev: v2.4.1 hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint - rev: v1.26.2 + rev: v1.26.3 hooks: - id: yamllint args: @@ -49,7 +49,7 @@ repos: # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v2.14.0 + rev: v2.15.0 hooks: - id: validate_manifest @@ -81,7 +81,7 @@ repos: args: - --config=.bandit.yml - repo: https://github.com/psf/black - rev: 21.7b0 + rev: 21.9b0 hooks: - id: black - repo: https://gitlab.com/pycqa/flake8 @@ -95,11 +95,11 @@ repos: hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.910 + rev: v0.910-1 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v2.23.3 + rev: v2.29.0 hooks: - id: pyupgrade @@ -114,7 +114,7 @@ repos: # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.50.0 + rev: v1.52.0 hooks: - id: terraform_fmt - id: terraform_validate From b78b83226904126138ad1bdc1fc37e44ed7c2d41 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 1 Oct 2021 13:01:56 -0400 Subject: [PATCH 268/282] Update the ansible-lint version for pre-commit --- .pre-commit-config.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3171404..d83fdbc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -105,9 +105,7 @@ repos: # Ansible hooks - repo: https://github.com/ansible-community/ansible-lint - # This is intentionally being held back because of issues in v5 per - # https://github.com/cisagov/skeleton-ansible-role/issues/69 - rev: v4.3.7 + rev: v5.2.1 hooks: - id: ansible-lint # files: molecule/default/playbook.yml From a3c5aa71cf2e8497bb0581b09bbf2978b6a96fc6 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 1 Oct 2021 13:05:45 -0400 Subject: [PATCH 269/282] Add an ansible-lint configuration file This file is copied from cisagov/skeleton-ansible-role#85 and tweaked to pass pre-commit. See here for for a list of the elements that can exist in this file: https://ansible-lint.readthedocs.io/en/latest/configuring.html Co-authored-by: Shane Frasier --- .ansible-lint | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .ansible-lint diff --git a/.ansible-lint b/.ansible-lint new file mode 100644 index 0000000..bc8e0e5 --- /dev/null +++ b/.ansible-lint @@ -0,0 +1,22 @@ +--- +# See https://ansible-lint.readthedocs.io/en/latest/configuring.html +# for a list of the configuration elements that can exist in this +# file. +enable_list: + # Useful checks that one must opt-into. See here for more details: + # https://ansible-lint.readthedocs.io/en/latest/rules.html + - fcqn-builtins + - no-log-password + - no-same-owner +exclude_paths: + # This exclusion is implicit, unless exclude_paths is defined + - .cache + # Seems wise to ignore this too + - .github + # ansible-lint doesn't like the role name in this playbook, but it's + # what molecule requires + - molecule/default/converge.yml + # These two are Molecule configuration files, not Ansible playbooks + - molecule/default/molecule-no-systemd.yml + - molecule/default/molecule-with-systemd.yml +use_default_rules: true From 24df40abd141eb93c056df90a7809fd8573d5328 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 8 Oct 2021 12:19:14 -0400 Subject: [PATCH 270/282] Add dependabot configuration for Terraform This will configure `dependabot` to scan Terraform configurations if they exist in a repository. If a repository stores a Terraform configuration in a sub- directory this configuration will need to be modified or an additional configuration must be added if there still exists a Terraform configuration in the root directory. --- .github/dependabot.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 1fd2ed3..a3bcd94 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -11,3 +11,8 @@ updates: directory: "/" schedule: interval: "weekly" + + - package-ecosystem: "terraform" + directory: "/" + schedule: + interval: "weekly" From 6cf78c02e11f78ebc35a151a38f4280bb0f8e53c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 30 Sep 2021 01:00:47 -0400 Subject: [PATCH 271/282] Use an id when using cisagov/setup-env-github-action --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 090f039..18a327f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,7 +17,8 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: cisagov/setup-env-github-action@develop + - id: setup-env + uses: cisagov/setup-env-github-action@develop - uses: actions/checkout@v2 - id: setup-python uses: actions/setup-python@v2 From 1e16136272bfd8d55dcb554c4c1ee059223fafe7 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 30 Sep 2021 11:33:00 -0400 Subject: [PATCH 272/282] Use setup-env outputs for Terraform version Change from using an environment variable to the outputs of the cisagov/setup-env-github-action instead. --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 18a327f..eb0f504 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -44,7 +44,7 @@ jobs: py${{ steps.setup-python.outputs.python-version }}-\ go${{ env.GO_VERSION }}-\ packer${{ env.PACKER_VERSION }}-\ - tf${{ env.TERRAFORM_VERSION }}-" + tf${{ steps.setup-env.outputs.terraform-version }}-" with: # Note that the .terraform directory IS NOT included in the # cache because if we were caching, then we would need to use @@ -78,7 +78,7 @@ jobs: sudo ln -s /opt/packer/packer /usr/local/bin/packer - uses: hashicorp/setup-terraform@v1 with: - terraform_version: ${{ env.TERRAFORM_VERSION }} + terraform_version: ${{ steps.setup-env.outputs.terraform-version }} - name: Install shfmt run: go install mvdan.cc/sh/v3/cmd/shfmt@${SHFMT_VERSION} - name: Install Terraform-docs From 64b24714ac3f06a2fc2f5d9096eca29ecac41e47 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 1 Oct 2021 11:03:19 -0400 Subject: [PATCH 273/282] Use setup-env outputs for Packer version We use a `PACKER_VERSION` environment variable for the "Install Packer" step that is populated from the `setup-env` outputs to get around `yamllint` lint length limits. --- .github/workflows/build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eb0f504..e6ef7ce 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,7 +43,7 @@ jobs: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ py${{ steps.setup-python.outputs.python-version }}-\ go${{ env.GO_VERSION }}-\ - packer${{ env.PACKER_VERSION }}-\ + packer${{ steps.setup-env.outputs.packer-version }}-\ tf${{ steps.setup-env.outputs.terraform-version }}-" with: # Note that the .terraform directory IS NOT included in the @@ -66,6 +66,8 @@ jobs: - name: Setup curl cache run: mkdir -p ${{ env.CURL_CACHE_DIR }} - name: Install Packer + env: + PACKER_VERSION: ${{ steps.setup-env.outputs.packer-version }} run: | PACKER_ZIP="packer_${PACKER_VERSION}_linux_amd64.zip" curl --output ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ From 0851598edda3bf21d0976b4c2fa615e15cbec8e4 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 1 Oct 2021 11:08:15 -0400 Subject: [PATCH 274/282] Use setup-env outputs for shfmt version We change the "Install shfmt" step to use two local environment variables to provide the package's URL and version to install. This allows us to work around `yamllint` line length limits. --- .github/workflows/build.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e6ef7ce..f0f39b2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -82,7 +82,10 @@ jobs: with: terraform_version: ${{ steps.setup-env.outputs.terraform-version }} - name: Install shfmt - run: go install mvdan.cc/sh/v3/cmd/shfmt@${SHFMT_VERSION} + env: + PACKAGE_URL: mvdan.cc/sh/v3/cmd/shfmt + PACKAGE_VERSION: ${{ steps.setup-env.outputs.shfmt-version }} + run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - name: Install Terraform-docs run: | go install \ From 2699e09eb7e81ccd73d5f668345ad711dc162dc5 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 1 Oct 2021 11:11:11 -0400 Subject: [PATCH 275/282] Use setup-env outputs for Terraform-docs version We change the "Install Terraform-docs" step to use two local environment variables to provide the package's URL and version to install. This allows us to work around `yamllint` line length limits. --- .github/workflows/build.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f0f39b2..c7141c1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -87,9 +87,10 @@ jobs: PACKAGE_VERSION: ${{ steps.setup-env.outputs.shfmt-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - name: Install Terraform-docs - run: | - go install \ - github.com/terraform-docs/terraform-docs@${TERRAFORM_DOCS_VERSION} + env: + PACKAGE_URL: github.com/terraform-docs/terraform-docs + PACKAGE_VERSION: ${{ steps.setup-env.outputs.terraform-docs-version }} + run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - name: Install dependencies run: | python -m pip install --upgrade pip From 8797e110d3c9cefbfe7e72a80496164bcdd7620a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 1 Oct 2021 11:49:45 -0400 Subject: [PATCH 276/282] Store Go version as a step output This changes from using an environment variable to using a step output to store the Go version that is installed. This mirrors changes made to the other program versions and how they're stored. --- .github/workflows/build.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c7141c1..7be2ce8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,16 +24,16 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.9 - # GO_VERSION and GOCACHE are used by the cache task, so the Go - # installation must happen before that. + # We need the Go version and Go cache location for the actions/cache step, + # so the Go installation must happen before that. - uses: actions/setup-go@v2 with: go-version: '1.16' - name: Store installed Go version + id: go-version run: | - echo "GO_VERSION="\ - "$(go version | sed 's/^go version go\([0-9.]\+\) .*/\1/')" \ - >> $GITHUB_ENV + echo "::set-output name=version::"\ + "$(go version | sed 's/^go version go\([0-9.]\+\) .*/\1/')" - name: Lookup Go cache directory id: go-cache run: | @@ -42,7 +42,7 @@ jobs: env: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ py${{ steps.setup-python.outputs.python-version }}-\ - go${{ env.GO_VERSION }}-\ + go${{ steps.go-version.outputs.version }}-\ packer${{ steps.setup-env.outputs.packer-version }}-\ tf${{ steps.setup-env.outputs.terraform-version }}-" with: From 6a10135f8155b0128a0019d451bc9673a0c463ee Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 27 Oct 2021 21:50:51 -0400 Subject: [PATCH 277/282] Update ansible-lint configuration Remove the manual exclusion of the `converge.yml` file in the default molecule scenario. With the changes in cisagov/skeleton-ansible-role#88 this exclusion is no longer necessary. --- .ansible-lint | 3 --- 1 file changed, 3 deletions(-) diff --git a/.ansible-lint b/.ansible-lint index bc8e0e5..08c1ae7 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -13,9 +13,6 @@ exclude_paths: - .cache # Seems wise to ignore this too - .github - # ansible-lint doesn't like the role name in this playbook, but it's - # what molecule requires - - molecule/default/converge.yml # These two are Molecule configuration files, not Ansible playbooks - molecule/default/molecule-no-systemd.yml - molecule/default/molecule-with-systemd.yml From 4b912fe7caf317793606afc626d19a4b840d4f2e Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 27 Oct 2021 21:53:10 -0400 Subject: [PATCH 278/282] Update ansible-lint configuration Instead of excluding molecule configurations for systemd individually we instead direct ansible-lint to process any of these configurations in scenario directories as plain yaml files. This mirrors how ansible-lint handles `molecule.yml` files in scenario directories by default. --- .ansible-lint | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.ansible-lint b/.ansible-lint index 08c1ae7..0e80b05 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -13,7 +13,10 @@ exclude_paths: - .cache # Seems wise to ignore this too - .github - # These two are Molecule configuration files, not Ansible playbooks - - molecule/default/molecule-no-systemd.yml - - molecule/default/molecule-with-systemd.yml +kinds: + # This will force our systemd specific molecule configurations to be treated + # as plain yaml files by ansible-lint. This mirrors the default kind + # configuration in ansible-lint for molecule configurations: + # yaml: "**/molecule/*/{base,molecule}.{yaml,yml}" + - yaml: "**/molecule/*/molecule-{no,with}-systemd.yml" use_default_rules: true From d0c117f1636b8b18ac675e71f5ffc8b0f4397ed6 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 29 Oct 2021 15:11:45 -0400 Subject: [PATCH 279/282] Disable fail-fast for matrix GHA jobs Override the default by setting `fail-fast` to `false` for the test and build jobs in our GHA build workflow. This ensure that all matrix jobs run regardless of a failure. --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a2ec81a..31d1120 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -109,6 +109,7 @@ jobs: test: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: python-version: - "3.6" @@ -195,6 +196,7 @@ jobs: runs-on: ubuntu-latest needs: [lint, test] strategy: + fail-fast: false matrix: python-version: - "3.6" From 3f370ebddeb80461a609e0ac1c51e4c1dfb04652 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 17 Nov 2021 12:22:15 -0500 Subject: [PATCH 280/282] Update pre-commit hooks Update pre-commit hooks using `pre-commit autoupdate`. --- .pre-commit-config.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d83fdbc..b8dcbe0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.0.1 + rev: v4.1.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs @@ -31,13 +31,13 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.29.0 + rev: v0.30.0 hooks: - id: markdownlint args: - --config=.mdl_config.yaml - repo: https://github.com/pre-commit/mirrors-prettier - rev: v2.4.1 + rev: v2.5.1 hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint @@ -49,7 +49,7 @@ repos: # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v2.15.0 + rev: v2.16.0 hooks: - id: validate_manifest @@ -75,13 +75,13 @@ repos: # Python hooks - repo: https://github.com/PyCQA/bandit - rev: 1.7.0 + rev: 1.7.1 hooks: - id: bandit args: - --config=.bandit.yml - repo: https://github.com/psf/black - rev: 21.9b0 + rev: 21.12b0 hooks: - id: black - repo: https://gitlab.com/pycqa/flake8 @@ -91,28 +91,28 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/PyCQA/isort - rev: 5.9.3 + rev: 5.10.1 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.910-1 + rev: v0.931 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v2.29.0 + rev: v2.31.0 hooks: - id: pyupgrade # Ansible hooks - repo: https://github.com/ansible-community/ansible-lint - rev: v5.2.1 + rev: v5.3.2 hooks: - id: ansible-lint # files: molecule/default/playbook.yml # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.52.0 + rev: v1.62.3 hooks: - id: terraform_fmt - id: terraform_validate From a019da15d020733697ebd737fb8faa8f2b6596b4 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sat, 5 Mar 2022 02:16:14 -0500 Subject: [PATCH 281/282] Update to satisfy pre-commit Perform all changes necessary to satisfy the pre-commit configuration. --- README.md | 8 ++++---- src/lcgit/lcgit.py | 5 +++-- tests/test_lcgit.py | 11 ++++++++--- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index b35765e..94aaf6a 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# lcgit 🎰 +# lcgit 🎰 # [![GitHub Build Status](https://github.com/cisagov/lcgit/workflows/build/badge.svg)](https://github.com/cisagov/lcgit/actions) [![Coverage Status](https://coveralls.io/repos/github/cisagov/lcgit/badge.svg?branch=develop)](https://coveralls.io/github/cisagov/lcgit?branch=develop) @@ -68,14 +68,14 @@ for i in lcg(range(100_000_000_000_000)): print(i) ``` -## NOOICE! 🕺 +## NOOICE! 🕺 ## -## Contributing +## Contributing ## We welcome contributions! Please see [`CONTRIBUTING.md`](CONTRIBUTING.md) for details. -## License +## License ## This project is in the worldwide [public domain](LICENSE). diff --git a/src/lcgit/lcgit.py b/src/lcgit/lcgit.py index db97f74..346870b 100755 --- a/src/lcgit/lcgit.py +++ b/src/lcgit/lcgit.py @@ -9,10 +9,11 @@ see: https://en.wikipedia.org/wiki/Linear_congruential_generator """ -from random import randint, shuffle -from math import sin +# Standard Python Libraries from collections.abc import Sequence from ipaddress import _BaseNetwork +from math import sin +from random import randint, shuffle def _lcg_params(u, v): diff --git a/tests/test_lcgit.py b/tests/test_lcgit.py index 0736f7c..4b077dd 100644 --- a/tests/test_lcgit.py +++ b/tests/test_lcgit.py @@ -1,8 +1,13 @@ #!/usr/bin/env pytest -vs """Tests for lcgit.""" -import pytest +# Standard Python Libraries from ipaddress import ip_network as net + +# Third-Party Libraries +import pytest + +# cisagov Libraries from lcgit import lcg, lcgit sequences = [ @@ -25,7 +30,7 @@ @pytest.mark.parametrize("sequence", sequences) def test_counts_and_dups(sequence): """Verify LCG output integrity.""" - answer = sorted([i for i in sequence]) + answer = sorted(i for i in sequence) my_lcg = lcg(sequence) accumulated = [] count = 0 @@ -40,7 +45,7 @@ def test_counts_and_dups(sequence): @pytest.mark.parametrize("sequence", sequences) def test_state_save_and_restore(sequence): """Verify state save and restore.""" - answer = sorted([i for i in sequence]) + answer = sorted(i for i in sequence) lcg1 = lcg(sequence, emit=True) accumulated = [] break_at = len(lcg1) / 2 From 45c26b47ab3d5732035c460f1c0440f52ce661e4 Mon Sep 17 00:00:00 2001 From: Nick <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 7 Mar 2022 10:09:45 -0500 Subject: [PATCH 282/282] Update missed skeleton reference Reference the correct path when bumping versions. Co-authored-by: dav3r --- bump_version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bump_version.sh b/bump_version.sh index e1324b8..704b162 100755 --- a/bump_version.sh +++ b/bump_version.sh @@ -6,7 +6,7 @@ set -o nounset set -o errexit set -o pipefail -VERSION_FILE=src/example/_version.py +VERSION_FILE=src/lcgit/_version.py HELP_INFORMATION="bump_version.sh (show|major|minor|patch|prerelease|build|finalize)"