diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index eb8d9b1f..81cc66b9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,7 +36,7 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/asottile/pyupgrade - rev: v2.29.1 + rev: v2.30.1 hooks: - id: pyupgrade - repo: https://github.com/psf/black @@ -66,6 +66,22 @@ repos: hooks: - id: prettier stages: [commit] + - repo: https://github.com/myint/autoflake + rev: v1.4 + hooks: + - id: autoflake + name: autoflake + language: python + args: + [ + --in-place, + --remove-all-unused-imports, + --remove-unused-variables, + --remove-duplicate-keys, + --ignore-init-module-imports, + --exclude, + compat.py, + ] - repo: https://github.com/PyCQA/flake8 rev: 4.0.1 hooks: @@ -88,6 +104,10 @@ repos: rev: v0.930 hooks: - id: mypy + args: [ + # https://mypy.readthedocs.io/en/stable/command_line.html#cmdoption-mypy-show-error-codes + --show-error-codes, + ] # Install additional types to fix new warnings that appeared on v0.910: # https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports # "using --install-types is problematic" diff --git a/.prettierignore b/.prettierignore index 0018b46e..988a563f 100644 --- a/.prettierignore +++ b/.prettierignore @@ -3,3 +3,6 @@ index.html _includes/* *.min.css README.md + +# Prettier uses double quotes, ruamel.yaml uses single quotes +tests/test_yaml/*.yaml diff --git a/.pylintrc b/.pylintrc index b9204800..7ede5b22 100644 --- a/.pylintrc +++ b/.pylintrc @@ -30,7 +30,7 @@ output-format=colorized # --disable=W" # Configurations for the black formatter disable=bad-continuation,bad-whitespace, - fixme,cyclic-import + fixme,cyclic-import,line-too-long [BASIC] diff --git a/Makefile b/Makefile index 864f30f5..5fb916d6 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ SRC := $(shell find docs src -type f -a -iname '*.py') DOCS := docs/*.rst *.rst *.md -STYLES := $(shell find styles -type f) +STYLES := $(shell find src/nitpick/resources -type f) TESTS := $(shell find tests -type f -iname '*.py') GITHUB = $(shell find .github -type f) ANY := $(SRC) $(DOCS) $(STYLES) $(TESTS) $(GITHUB) diff --git a/README.rst b/README.rst index 6e7b496c..08016597 100644 --- a/README.rst +++ b/README.rst @@ -109,6 +109,9 @@ Implemented * - `Any TOML file `_ - ✅ - ✅ + * - `Any YAML file (except .pre-commit-config.yaml) `_ + - ✅ + - ✅ * - `.editorconfig `_ - ✅ - ✅ @@ -232,8 +235,6 @@ Run as a pre-commit hook If you use `pre-commit `_ on your project, add this to the ``.pre-commit-config.yaml`` in your repository:: -.. code-block:: yaml - repos: - repo: https://github.com/andreoliwa/nitpick rev: v0.29.0 @@ -247,8 +248,6 @@ There are 3 available hook IDs: If you want to run Nitpick as a flake8 plugin instead:: -.. code-block:: yaml - repos: - repo: https://github.com/PyCQA/flake8 rev: 4.0.1 diff --git a/docs/conf.py b/docs/conf.py index 682f50f1..d205a1fa 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -112,7 +112,7 @@ (key, identifier) for key, identifiers in { "py:class": { - "BaseFormat", + "BaseDoc", "bool|tuple", "builtins.dict", "callable", diff --git a/docs/examples.rst b/docs/examples.rst index f755f787..6d9a164e 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -13,70 +13,48 @@ You can use these examples directly with their URL (see :ref:`multiple_styles`), .. auto-generated-from-here -.. _example-absent-files: - -Absent files ------------- - -Contents of `styles/absent-files.toml `_: - -.. code-block:: toml - - [nitpick.files.absent] - "requirements.txt" = "Install poetry, run 'poetry init' to create pyproject.toml, and move dependencies to it" - ".isort.cfg" = "Move values to setup.cfg, section [isort]" - "Pipfile" = "Use pyproject.toml instead" - "Pipfile.lock" = "Use pyproject.toml instead" - ".venv" = "" - ".pyup.yml" = "Configure safety instead: https://github.com/pyupio/safety#using-safety-with-a-ci-service" - -.. _example-black: +.. _example-commitizen: -black_ ------- +commitizen_ +----------- -Contents of `styles/black.toml `_: +Contents of `resources/any/commitizen.toml `_: .. code-block:: toml - ["pyproject.toml".tool.black] - line-length = 120 - [[".pre-commit-config.yaml".repos]] yaml = """ - - repo: https://github.com/psf/black - hooks: - - id: black - args: [--safe, --quiet] - - repo: https://github.com/asottile/blacken-docs + - repo: https://github.com/commitizen-tools/commitizen hooks: - - id: blacken-docs - additional_dependencies: [black==21.5b2] + - id: commitizen + stages: [commit-msg] """ - # TODO The toml library has issues loading arrays with multiline strings: - # https://github.com/uiri/toml/issues/123 - # https://github.com/uiri/toml/issues/230 - # If they are fixed one day, remove this 'yaml' key and use only a 'repos' list with a single element: - #[".pre-commit-config.yaml"] - #repos = [""" - # - #"""] -.. _example-commitizen: +.. _example-commitlint: -commitizen_ +commitlint_ ----------- -Contents of `styles/commitizen.toml `_: +Contents of `resources/any/commitlint.toml `_: -.. code-block:: toml +.. code-block:: + + ["package.json".contains_json] + commitlint = """ + { + "extends": [ + "@commitlint/config-conventional" + ] + } + """ [[".pre-commit-config.yaml".repos]] yaml = """ - - repo: https://github.com/commitizen-tools/commitizen + - repo: https://github.com/alessandrojcm/commitlint-pre-commit-hook hooks: - - id: commitizen + - id: commitlint stages: [commit-msg] + additional_dependencies: ['@commitlint/config-conventional'] """ .. _example-editorconfig: @@ -84,7 +62,7 @@ Contents of `styles/commitizen.toml `_: +Contents of `resources/any/editorconfig.toml `_: .. code-block:: toml @@ -119,12 +97,96 @@ Contents of `styles/editorconfig.toml `_: + +.. code-block:: toml + + # See https://pre-commit.com for more information + # See https://pre-commit.com/hooks.html for more hooks + + [nitpick.files.present] + ".pre-commit-config.yaml" = "Create the file with the contents below, then run 'pre-commit install'" + + [[".pre-commit-config.yaml".repos]] + yaml = """ + - repo: https://github.com/pre-commit/pre-commit-hooks + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + """ + +.. _example-package-json: + +package.json_ +------------- + +Contents of `resources/javascript/package-json.toml `_: + +.. code-block:: toml + + ["package.json"] + contains_keys = ["name", "version", "repository.type", "repository.url", "release.plugins"] + +.. _example-absent-files: + +Absent files +------------ + +Contents of `resources/python/absent.toml `_: + +.. code-block:: toml + + [nitpick.files.absent] + "requirements.txt" = "Install poetry, run 'poetry init' to create pyproject.toml, and move dependencies to it" + ".isort.cfg" = "Move values to setup.cfg, section [isort]" + "Pipfile" = "Use pyproject.toml instead" + "Pipfile.lock" = "Use pyproject.toml instead" + ".venv" = "" + ".pyup.yml" = "Configure safety instead: https://github.com/pyupio/safety#using-safety-with-a-ci-service" + +.. _example-black: + +black_ +------ + +Contents of `resources/python/black.toml `_: + +.. code-block:: toml + + ["pyproject.toml".tool.black] + line-length = 120 + + [[".pre-commit-config.yaml".repos]] + yaml = """ + - repo: https://github.com/psf/black + hooks: + - id: black + args: [--safe, --quiet] + - repo: https://github.com/asottile/blacken-docs + hooks: + - id: blacken-docs + additional_dependencies: [black==21.5b2] + """ + # TODO The toml library has issues loading arrays with multiline strings: + # https://github.com/uiri/toml/issues/123 + # https://github.com/uiri/toml/issues/230 + # If they are fixed one day, remove this 'yaml' key and use only a 'repos' list with a single element: + #[".pre-commit-config.yaml"] + #repos = [""" + # + #"""] + .. _example-flake8: flake8_ ------- -Contents of `styles/flake8.toml `_: +Contents of `resources/python/flake8.toml `_: .. code-block:: toml @@ -162,12 +224,38 @@ Contents of `styles/flake8.toml `_: + +.. code-block:: toml + + [[".pre-commit-config.yaml".repos]] + yaml = """ + - repo: https://github.com/pre-commit/pygrep-hooks + hooks: + - id: python-check-blanket-noqa + - id: python-check-mock-methods + - id: python-no-eval + - id: python-no-log-warn + - id: rst-backticks + - repo: https://github.com/pre-commit/pre-commit-hooks + hooks: + - id: debug-statements + - repo: https://github.com/asottile/pyupgrade + hooks: + - id: pyupgrade + """ + .. _example-ipython: IPython_ -------- -Contents of `styles/ipython.toml `_: +Contents of `resources/python/ipython.toml `_: .. code-block:: toml @@ -180,7 +268,7 @@ Contents of `styles/ipython.toml `_: +Contents of `resources/python/isort.toml `_: .. code-block:: toml @@ -209,7 +297,7 @@ Contents of `styles/isort.toml `_: +Contents of `resources/python/mypy.toml `_: .. code-block:: toml @@ -217,8 +305,8 @@ Contents of `styles/mypy.toml `_: - -.. code-block:: toml - - ["package.json"] - contains_keys = ["name", "version", "repository.type", "repository.url", "release.plugins"] - .. _example-poetry: Poetry_ ------- -Contents of `styles/poetry.toml `_: +Contents of `resources/python/poetry.toml `_: .. code-block:: toml [nitpick.files.present] "pyproject.toml" = "Install poetry and run 'poetry init' to create it" -.. _example-bash: - -Bash_ ------ - -Contents of `styles/pre-commit/bash.toml `_: - -.. code-block:: toml - - [[".pre-commit-config.yaml".repos]] - yaml = """ - - repo: https://github.com/openstack/bashate - hooks: - - id: bashate - """ - -.. _example-pre-commit-hooks: - -pre-commit_ (hooks) -------------------- - -Contents of `styles/pre-commit/general.toml `_: - -.. code-block:: toml - - [[".pre-commit-config.yaml".repos]] - yaml = """ - - repo: https://github.com/pre-commit/pre-commit-hooks - hooks: - - id: end-of-file-fixer - - id: trailing-whitespace - """ - -.. _example-pre-commit-main: - -pre-commit_ (main) ------------------- - -Contents of `styles/pre-commit/main.toml `_: - -.. code-block:: toml - - # See https://pre-commit.com for more information - # See https://pre-commit.com/hooks.html for more hooks - - [nitpick.files.present] - ".pre-commit-config.yaml" = "Create the file with the contents below, then run 'pre-commit install'" - -.. _example-pre-commit-python-hooks: - -pre-commit_ (Python hooks) --------------------------- - -Contents of `styles/pre-commit/python.toml `_: - -.. code-block:: toml - - [[".pre-commit-config.yaml".repos]] - yaml = """ - - repo: https://github.com/pre-commit/pygrep-hooks - hooks: - - id: python-check-blanket-noqa - - id: python-check-mock-methods - - id: python-no-eval - - id: python-no-log-warn - - id: rst-backticks - - repo: https://github.com/pre-commit/pre-commit-hooks - hooks: - - id: debug-statements - - repo: https://github.com/asottile/pyupgrade - hooks: - - id: pyupgrade - """ - .. _example-pylint: Pylint_ ------- -Contents of `styles/pylint.toml `_: +Contents of `resources/python/pylint.toml `_: .. code-block:: toml @@ -366,7 +369,7 @@ Contents of `styles/pylint.toml `_: +Contents of `resources/python/python310.toml `_: .. code-block:: toml @@ -416,7 +419,7 @@ Contents of `styles/python310.toml `_: +Contents of `resources/python/python36.toml `_: .. code-block:: toml @@ -428,7 +431,7 @@ Contents of `styles/python36.toml `_: +Contents of `resources/python/python37.toml `_: .. code-block:: toml @@ -440,7 +443,7 @@ Contents of `styles/python37.toml `_: +Contents of `resources/python/python38.toml `_: .. code-block:: toml @@ -452,7 +455,7 @@ Contents of `styles/python38.toml `_: +Contents of `resources/python/python39.toml `_: .. code-block:: toml @@ -464,7 +467,7 @@ Contents of `styles/python39.toml `_: +Contents of `resources/python/tox.toml `_: .. code-block:: toml @@ -496,3 +499,19 @@ Contents of `styles/tox.toml `_: + +.. code-block:: toml + + [[".pre-commit-config.yaml".repos]] + yaml = """ + - repo: https://github.com/openstack/bashate + hooks: + - id: bashate + """ diff --git a/docs/generate_rst.py b/docs/generate_rst.py index faec0a0c..b06102da 100644 --- a/docs/generate_rst.py +++ b/docs/generate_rst.py @@ -42,31 +42,31 @@ MD_DIVIDER_END = "" DOCS_DIR: Path = Path(__file__).parent.absolute() -STYLES_DIR: Path = DOCS_DIR.parent / "styles" +STYLES_DIR: Path = DOCS_DIR.parent / "src" / "nitpick" / "resources" STYLE_MAPPING = SortedDict( { - "absent-files.toml": "Absent files", - "black.toml": "black_", - "editorconfig.toml": "EditorConfig_", - "flake8.toml": "flake8_", - "ipython.toml": "IPython_", - "isort.toml": "isort_", - "mypy.toml": "mypy_", - "package-json.toml": "package.json_", - "poetry.toml": "Poetry_", - "pre-commit/bash.toml": "Bash_", - "commitizen.toml": "commitizen_", - "pre-commit/general.toml": "pre-commit_ (hooks)", - "pre-commit/main.toml": "pre-commit_ (main)", - "pre-commit/python.toml": "pre-commit_ (Python hooks)", - "pylint.toml": "Pylint_", - "python36.toml": "Python 3.6", - "python37.toml": "Python 3.7", - "python38.toml": "Python 3.8", - "python39.toml": "Python 3.9", - "python310.toml": "Python 3.10", - "tox.toml": "tox_", + "python/absent.toml": "Absent files", + "python/black.toml": "black_", + "any/editorconfig.toml": "EditorConfig_", + "python/flake8.toml": "flake8_", + "python/ipython.toml": "IPython_", + "python/isort.toml": "isort_", + "python/mypy.toml": "mypy_", + "javascript/package-json.toml": "package.json_", + "python/poetry.toml": "Poetry_", + "shell/hooks.toml": "Bash_", + "any/commitizen.toml": "commitizen_", + "any/commitlint.toml": "commitlint_", + "any/hooks.toml": "pre-commit_ (hooks)", + "python/hooks.toml": "pre-commit_ (Python hooks)", + "python/pylint.toml": "Pylint_", + "python/python36.toml": "Python 3.6", + "python/python37.toml": "Python 3.7", + "python/python38.toml": "Python 3.8", + "python/python39.toml": "Python 3.9", + "python/python310.toml": "Python 3.10", + "python/tox.toml": "tox_", } ) CLI_MAPPING = [ @@ -94,7 +94,7 @@ class FileType: text: str url: str check: Union[bool, int] - fix: Union[bool, int] + autofix: Union[bool, int] def __post_init__(self): """Warn about text that might render incorrectly.""" @@ -136,14 +136,14 @@ def check_str(self) -> str: return self._pretty("check") @property - def fix_str(self) -> str: - """The fix flag, as a string.""" - return self._pretty("fix") + def autofix_str(self) -> str: + """The autofix flag, as a string.""" + return self._pretty("autofix") @property def row(self) -> Tuple[str, str, str]: """Tuple for a table row.""" - return self.text_with_url, self.check_str, self.fix_str + return self.text_with_url, self.check_str, self.autofix_str IMPLEMENTED_FILE_TYPES: Set[FileType] = { @@ -151,6 +151,9 @@ def row(self) -> Tuple[str, str, str]: FileType("Any JSON file", f"{READ_THE_DOCS_URL}plugins.html#json-files", True, True), FileType("Any text file", f"{READ_THE_DOCS_URL}plugins.html#text-files", True, False), FileType("Any TOML file", f"{READ_THE_DOCS_URL}plugins.html#toml-files", True, True), + FileType( + f"Any YAML file (except {PRE_COMMIT_CONFIG_YAML})", f"{READ_THE_DOCS_URL}plugins.html#yaml-files", True, True + ), FileType(EDITOR_CONFIG, f"{READ_THE_DOCS_URL}examples.html#example-editorconfig", True, True), FileType(PRE_COMMIT_CONFIG_YAML, f"{READ_THE_DOCS_URL}plugins.html#pre-commit-config-yaml", True, 282), FileType(PYLINTRC, f"{READ_THE_DOCS_URL}plugins.html#ini-files", True, True), diff --git a/docs/ideas/lab.py b/docs/ideas/lab.py index 9ca85fc2..dc2c1c54 100644 --- a/docs/ideas/lab.py +++ b/docs/ideas/lab.py @@ -6,16 +6,16 @@ import click import jmespath -from nitpick.formats import TOMLFormat, YAMLFormat +from nitpick.documents import TomlDoc, YamlDoc from nitpick.generic import flatten, search_dict -workflow = YAMLFormat(path=Path(".github/workflows/python.yaml")) +workflow = YamlDoc(path=Path(".github/workflows/python.yaml")) def find(expression): """Find with JMESpath.""" print(f"\nExpression: {expression}") - rv = search_dict(jmespath.compile(expression), workflow.as_data, {}) + rv = search_dict(jmespath.compile(expression), workflow.as_object, {}) print(f"Type: {type(rv)}") pprint(rv) @@ -25,8 +25,8 @@ def main(): for path in sorted(Path("docs/ideas/yaml").glob("*.toml")): click.secho(str(path), fg="green") - toml_format = TOMLFormat(path=path) - config: dict = toml_format.as_data[".github/workflows/python.yaml"] + toml_doc = TomlDoc(path=path) + config: dict = toml_doc.as_object[".github/workflows/python.yaml"] # config.pop("contains") # config.pop("contains_sorted") diff --git a/docs/ideas/yaml/jmespath-on-section.toml b/docs/ideas/yaml/jmespath-on-section.toml deleted file mode 100644 index 0876ec60..00000000 --- a/docs/ideas/yaml/jmespath-on-section.toml +++ /dev/null @@ -1,25 +0,0 @@ -# The values below were taken from .github/workflows/python.yaml in this repo - -# 1. JMESPath as part of the section name, after the file name. -# Everything after the file name is considered a JMESPath https://jmespath.org/ -# Format: ["path/to/file.ext".jmes.path.expression] -# -# 2. "jobs.build.strategy.matrix" should have "os" and "python-version" -# 3. Both are lists, and they have to be exactly as described here. -[".github/workflows/python.yaml".jobs.build.strategy.matrix] -os = ["ubuntu-latest", "macos-latest", "windows-latest"] -"python-version" = ["3.6", "3.7", "3.8", "3.9", "3.10"] - -# 4. "jobs.build" should have "runs-on" with value "${{ matrix.os }}" -[".github/workflows/python.yaml".jobs.build] -"runs-on" = "${{ matrix.os }}" - -# 5. "{{" and "}}" will conflict with Jinja https://github.com/andreoliwa/nitpick/issues/283 -# So we need a way to turn on/off Jinja templating. -# Probably "false" will be the default, to keep compatibility. -# Whoever wants to use Jinja will need to set "true" either here or as a global config on .nitpick.toml -__jinja = false - -# 6. Another way to turn off Jinja for a specific key only, not the whole dict -# (using the "__" syntax from Django filters, SQLAlchemy, factoryboy...) -"runs-on__no_jinja" = "${{ matrix.os }}" diff --git a/docs/ideas/yaml/jmespath-simple.toml b/docs/ideas/yaml/jmespath-simple.toml deleted file mode 100644 index 69300465..00000000 --- a/docs/ideas/yaml/jmespath-simple.toml +++ /dev/null @@ -1,7 +0,0 @@ -# Simplified API, having JMESPath as direct keys -# Read the discussion: https://github.com/andreoliwa/nitpick/pull/353/files#r613816390 -[".github/workflows/python.yaml"] -"jobs.build.strategy.matrix.os" = "foo" -"jobs.build.steps" = ["bar"] -"jobs.build.steps.regex" = "baz d+" -"jobs.build.steps.contains" = "baz" diff --git a/docs/ideas/yaml/jmespath-table.toml b/docs/ideas/yaml/jmespath-table.toml deleted file mode 100644 index cc74ce99..00000000 --- a/docs/ideas/yaml/jmespath-table.toml +++ /dev/null @@ -1,25 +0,0 @@ -# 1. Clean approach with JMESPath in tables and no reserved keys (`jmespath` or `__jmespath`) -# https://github.com/andreoliwa/nitpick/pull/353/files#r614633283 -[[".github/workflows/python.yaml".jobs.build.steps]] -uses = "actions/checkout@v2" - -[[".github/workflows/python.yaml".jobs.build.steps]] -name = "Set up Python ${{ matrix.python-version }}" -uses = "actions/setup-python@v2" -with = {"python-version" = "${{ matrix.python-version }}"} - -# 2. Complex JMESPath expressions should be quoted -# (I still don't know how to deal with JMESPath that matches multiple items) -[[".github/workflows/python.yaml"."jobs.build.steps[].{name: name, uses: uses}"]] -uses = "actions/checkout@v2" - -# 3. JMESPath expression that has double quotes, wrapped in single quotes for TOML -[[".github/workflows/python.yaml".'jobs.build.strategy.matrix."python-version"']] -name = "Set up Python ${{ matrix.python-version }}" -uses = "actions/setup-python@v2" -with = {"python-version" = "${{ matrix.python-version }}"} - -# 4. And it allows Jinja tuning in https://github.com/andreoliwa/nitpick/issues/283 -name__jinja = "Set up Python ${{ matrix.python-version }}" -name__no_jinja = "Set up Python ${{ matrix.python-version }}" -name__jinja_off = "Set up Python ${{ matrix.python-version }}" diff --git a/docs/ideas/yaml/jmespath.toml b/docs/ideas/yaml/jmespath.toml new file mode 100644 index 00000000..a4ddc6fb --- /dev/null +++ b/docs/ideas/yaml/jmespath.toml @@ -0,0 +1,39 @@ +# JMESPath as part of the section name, after the file name. +# Everything after the file name is considered a JMESPath https://jmespath.org/ +# Format: ["path/to/file.ext".jmes.path.expression] +# The values below were taken from .github/workflows/python.yaml in this repo + +# 1. Complex JMESPath expressions should be quoted +# (I still don't know how to deal with JMESPath that matches multiple items) +[[".github/workflows/python.yaml"."jobs.build.steps[].{name: name, uses: uses}"]] +uses = "actions/checkout@v2" + +# 2. JMESPath expression that has double quotes, wrapped in single quotes for TOML +[[".github/workflows/python.yaml".'jobs.build.strategy.matrix."python-version"']] +name = "Set up Python ${{ matrix.python-version }}" +uses = "actions/setup-python@v2" +with = {"python-version" = "${{ matrix.python-version }}"} + +# 3. It allows Jinja tuning in https://github.com/andreoliwa/nitpick/issues/283 +name__jinja = "Set up Python ${{ matrix.python-version }}" +name__no_jinja = "Set up Python ${{ matrix.python-version }}" +name__jinja_off = "Set up Python ${{ matrix.python-version }}" + +# 4. "{{" and "}}" will conflict with Jinja https://github.com/andreoliwa/nitpick/issues/283 +# So we need a way to turn on/off Jinja templating. +# Probably "false" will be the default, to keep compatibility. +# Whoever wants to use Jinja will need to set "true" either here or as a global config on .nitpick.toml +[".github/workflows/python.yaml".jobs.build] +__jinja = false + +# 5. Another way to turn off Jinja for a specific key only, not the whole dict +# (using the "__" syntax from Django filters, SQLAlchemy, factoryboy...) +"runs-on__no_jinja" = "${{ matrix.os }}" + +# 6. Simplified API, having JMESPath as direct keys +# Read the discussion: https://github.com/andreoliwa/nitpick/pull/353/files#r613816390 +[".github/workflows/jmespath-simple.yaml"] +"jobs.build.strategy.matrix.os" = "foo" +"jobs.build.steps" = ["bar"] +"jobs.build.steps.regex" = "baz d+" +"jobs.build.steps.contains" = "baz" diff --git a/docs/plugins.rst b/docs/plugins.rst index 132fdc6c..668ecd14 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -26,7 +26,7 @@ Style example: :ref:`the default pre-commit hooks `. INI files --------- -Enforce config on INI files. +Enforce configurations and autofix INI files. Examples of ``.ini`` files handled by this plugin: @@ -42,7 +42,7 @@ Style examples enforcing values on INI files: :ref:`flake8 configuration `. @@ -69,7 +69,7 @@ To check if ``some.txt`` file contains the lines ``abc`` and ``def`` (in any ord TOML files ---------- -Enforce config on TOML files. +Enforce configurations and autofix TOML files. E.g.: `pyproject.toml (PEP 518) `_. @@ -78,3 +78,10 @@ See also `the [tool.poetry] section of the pyproject.toml file Style example: :ref:`Python 3.8 version constraint `. There are :ref:`many other examples here `. + +.. _yamlplugin: + +YAML files +---------- + +Enforce configurations and autofix YAML files. diff --git a/docs/targets.rst b/docs/targets.rst index 484901d7..aaf7603b 100644 --- a/docs/targets.rst +++ b/docs/targets.rst @@ -8,6 +8,7 @@ .. _Bash: https://www.gnu.org/software/bash/ .. _black: https://github.com/psf/black .. _commitizen: https://github.com/commitizen-tools/commitizen +.. _commitlint: https://commitlint.js.org/ .. _Django: https://github.com/django/django .. _EditorConfig: https://editorconfig.org .. _flake8: https://github.com/PyCQA/flake8 diff --git a/nitpick-style.toml b/nitpick-style.toml index 3243dd42..be1c5d67 100644 --- a/nitpick-style.toml +++ b/nitpick-style.toml @@ -6,22 +6,21 @@ minimum_version = "0.10.0" [nitpick.styles] include = [ - "styles/python37", - "styles/poetry", - "styles/absent-files", - "styles/pre-commit/main", - "styles/pre-commit/general", - "styles/pre-commit/python", - "styles/pre-commit/bash", - "styles/commitizen", - "styles/black", - "styles/flake8", - "styles/isort", - "styles/mypy", - "styles/pylint", - "styles/tox", - "styles/package-json", - "styles/editorconfig", + "pypackage://nitpick/resources/python/python37", + "pypackage://nitpick/resources/python/poetry", + "pypackage://nitpick/resources/python/absent", + "pypackage://nitpick/resources/any/hooks", + "pypackage://nitpick/resources/python/hooks", + "pypackage://nitpick/resources/shell/hooks", + "pypackage://nitpick/resources/any/commitizen", + "pypackage://nitpick/resources/python/black", + "pypackage://nitpick/resources/python/flake8", + "pypackage://nitpick/resources/python/isort", + "pypackage://nitpick/resources/python/mypy", + "pypackage://nitpick/resources/python/pylint", + "pypackage://nitpick/resources/python/tox", + "pypackage://nitpick/resources/javascript/package-json", + "pypackage://nitpick/resources/any/editorconfig", ] [nitpick.files."setup.cfg"] diff --git a/poetry.lock b/poetry.lock index c251c594..1d477d4a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -758,7 +758,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "pygments" -version = "2.10.0" +version = "2.11.0" description = "Pygments is a syntax highlighting package written in Python." category = "main" optional = false @@ -1728,8 +1728,8 @@ pyflakes = [ {file = "pyflakes-2.4.0.tar.gz", hash = "sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c"}, ] pygments = [ - {file = "Pygments-2.10.0-py3-none-any.whl", hash = "sha256:b8e67fe6af78f492b3c4b3e2970c0624cbf08beb1e493b2c99b9fa1b67a20380"}, - {file = "Pygments-2.10.0.tar.gz", hash = "sha256:f398865f7eb6874156579fdf36bc840a03cab64d1cde9e93d68f46a425ec52c6"}, + {file = "Pygments-2.11.0-py3-none-any.whl", hash = "sha256:ac8098bfc40b8e1091ad7c13490c7f4797e401d0972e8fcfadde90ffb3ed4ea9"}, + {file = "Pygments-2.11.0.tar.gz", hash = "sha256:51130f778a028f2d19c143fce00ced6f8b10f726e17599d7e91b290f6cbcda0c"}, ] pylint = [ {file = "pylint-2.12.0-py3-none-any.whl", hash = "sha256:ba00afcb1550bc217bbcb0eb76c10cb8335f7417a3323bdd980c29fb5b59f8d2"}, diff --git a/pyproject.toml b/pyproject.toml index 1ef46268..c78c1194 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.nitpick] # Use the default style and override some things (like the Python version) -style = ["nitpick-style", "styles/python36"] +style = ["nitpick-style", "pypackage://nitpick/resources/python/python36.toml"] [tool.black] line-length = 120 @@ -38,9 +38,10 @@ NIP = "nitpick.flake8:NitpickFlake8Extension" [tool.poetry.plugins.nitpick] text = "nitpick.plugins.text" json = "nitpick.plugins.json" -pre_commit = "nitpick.plugins.pre_commit" +pre_commit = "nitpick.plugins.pre_commit" # TODO: remove this when removing the plugin class ini = "nitpick.plugins.ini" toml = "nitpick.plugins.toml" +yaml = "nitpick.plugins.yaml" [tool.poetry.dependencies] python = "^3.6.1" diff --git a/setup.cfg b/setup.cfg index 4afdf778..fc58d8a0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,9 +1,13 @@ [flake8] +# https://flake8.pycqa.org/en/latest/user/options.html ignore = D107,D401,D202,D203,E203,E402,E501,W503 max-line-length = 120 exclude = docs,.tox,build max-complexity = 10 inline-quotes = double +per-file-ignores = + # Imported but unused + compat.py:F401 # https://github.com/asottile/flake8-typing-imports#configuration min_python_version = 3.6.0 @@ -22,11 +26,15 @@ known_first_party = tests,nitpick [mypy] # https://mypy.readthedocs.io/en/stable/config_file.html ignore_missing_imports = True -follow_imports = skip +# https://mypy.readthedocs.io/en/stable/running_mypy.html#follow-imports +follow_imports = normal strict_optional = True warn_no_return = True warn_redundant_casts = True -warn_unused_ignores = True +# False positives when running on local machine... it works on pre-commit.ci ¯\_(ツ)_/¯ +warn_unused_ignores = false +exclude = + src/nitpick/compat.py [bandit] exclude = tests/* diff --git a/src/nitpick/cli.py b/src/nitpick/cli.py index 47f88cef..a47156ea 100644 --- a/src/nitpick/cli.py +++ b/src/nitpick/cli.py @@ -23,9 +23,9 @@ from nitpick.constants import PROJECT_NAME, TOOL_KEY, TOOL_NITPICK_KEY from nitpick.core import Nitpick +from nitpick.documents import TomlDoc from nitpick.enums import OptionEnum from nitpick.exceptions import QuitComplainingError -from nitpick.formats import TOMLFormat from nitpick.generic import relative_to_current_dir from nitpick.violations import Reporter @@ -64,7 +64,7 @@ def get_nitpick(context: click.Context) -> Nitpick: def common_fix_or_check(context, verbose: int, files, check_only: bool) -> None: - """Common CLI code for both fix and check commands.""" + """Common CLI code for both "fix" and "check" commands.""" if verbose: level = logging.INFO if verbose == 1 else logging.DEBUG @@ -77,7 +77,7 @@ def common_fix_or_check(context, verbose: int, files, check_only: bool) -> None: nit = get_nitpick(context) try: - for fuss in nit.run(*files, fix=not check_only): + for fuss in nit.run(*files, autofix=not check_only): nit.echo(fuss.pretty) except QuitComplainingError as err: for fuss in err.violations: @@ -147,7 +147,7 @@ def init(context): nit = get_nitpick(context) config = nit.project.read_configuration() - if config.file and PROJECT_NAME in TOMLFormat(path=config.file).as_data[TOOL_KEY]: + if config.file and PROJECT_NAME in TomlDoc(path=config.file).as_object[TOOL_KEY]: click.secho(f"The config file {config.file.name} already has a [{TOOL_NITPICK_KEY}] section.", fg="yellow") raise Exit(1) diff --git a/src/nitpick/compat.py b/src/nitpick/compat.py new file mode 100644 index 00000000..8b6f2f6d --- /dev/null +++ b/src/nitpick/compat.py @@ -0,0 +1,8 @@ +"""Handle import compatibility issues.""" +# pylint: skip-file +try: + from importlib.abc import Traversable # type: ignore[attr-defined] + from importlib.resources import files # type: ignore[attr-defined] +except ImportError: + from importlib_resources import files + from importlib_resources.abc import Traversable diff --git a/src/nitpick/constants.py b/src/nitpick/constants.py index 52b9c963..e8580857 100644 --- a/src/nitpick/constants.py +++ b/src/nitpick/constants.py @@ -54,7 +54,7 @@ DOUBLE_QUOTE = '"' #: Special unique separator for :py:meth:`flatten()` and :py:meth:`unflatten()`, -# to avoid collision with existing key values (e.g. the default dot separator "." can be part of a pyproject.toml key). +# to avoid collision with existing key values (e.g. the default dot separator "." can be part of a TOML key). SEPARATOR_FLATTEN = "$#@" #: Special unique separator for :py:meth:`nitpick.generic.quoted_split()`. diff --git a/src/nitpick/core.py b/src/nitpick/core.py index b1f62166..9e4b25bc 100644 --- a/src/nitpick/core.py +++ b/src/nitpick/core.py @@ -50,11 +50,11 @@ def init(self, project_root: PathOrStr = None, offline: bool = None) -> "Nitpick return self - def run(self, *partial_names: str, fix=False) -> Iterator[Fuss]: + def run(self, *partial_names: str, autofix=False) -> Iterator[Fuss]: """Run Nitpick. :param partial_names: Names of the files to enforce configs for. - :param fix: Flag to modify files, if the plugin supports it (default: True). + :param autofix: Flag to modify files, if the plugin supports it (default: True). :return: Fuss generator. """ Reporter.reset() @@ -63,7 +63,7 @@ def run(self, *partial_names: str, fix=False) -> Iterator[Fuss]: yield from chain( self.project.merge_styles(self.offline), self.enforce_present_absent(*partial_names), - self.enforce_style(*partial_names, fix=fix), + self.enforce_style(*partial_names, autofix=autofix), ) except QuitComplainingError as err: yield from err.violations @@ -95,14 +95,14 @@ def enforce_present_absent(self, *partial_names: str) -> Iterator[Fuss]: violation = ProjectViolations.MISSING_FILE if present else ProjectViolations.FILE_SHOULD_BE_DELETED yield reporter.make_fuss(violation, extra=extra) - def enforce_style(self, *partial_names: str, fix=True) -> Iterator[Fuss]: + def enforce_style(self, *partial_names: str, autofix=True) -> Iterator[Fuss]: """Read the merged style and enforce the rules in it. 1. Get all root keys from the merged style (every key is a filename, except "nitpick"). 2. For each file name, find the plugin(s) that can handle the file. :param partial_names: Names of the files to enforce configs for. - :param fix: Flag to modify files, if the plugin supports it (default: True). + :param autofix: Flag to modify files, if the plugin supports it (default: True). :return: Fuss generator. """ @@ -115,7 +115,7 @@ def enforce_style(self, *partial_names: str, fix=True) -> Iterator[Fuss]: info = FileInfo.create(self.project, config_key) # pylint: disable=no-member for plugin_class in self.project.plugin_manager.hook.can_handle(info=info): # type: Type[NitpickPlugin] - yield from plugin_class(info, config_dict, fix).entry_point() + yield from plugin_class(info, config_dict, autofix).entry_point() def configured_files(self, *partial_names: str) -> List[Path]: """List of files configured in the Nitpick style. Filter only the selected partial names.""" diff --git a/src/nitpick/formats.py b/src/nitpick/documents.py similarity index 54% rename from src/nitpick/formats.py rename to src/nitpick/documents.py index 4d45aa15..8fe7da6a 100644 --- a/src/nitpick/formats.py +++ b/src/nitpick/documents.py @@ -1,6 +1,5 @@ """Configuration file formats.""" import abc -import io import json from collections import OrderedDict from pathlib import Path @@ -8,14 +7,17 @@ import dictdiffer import toml +import tomlkit from autorepr import autorepr from loguru import logger -from ruamel.yaml import YAML, RoundTripRepresenter +from ruamel.yaml import YAML, RoundTripRepresenter, StringIO from ruamel.yaml.comments import CommentedMap, CommentedSeq from sortedcontainers import SortedDict from nitpick.generic import flatten, unflatten -from nitpick.typedefs import JsonDict, PathOrStr, YamlData +from nitpick.typedefs import JsonDict, PathOrStr, YamlObject, YamlValue + +DictOrYamlObject = Union[JsonDict, YamlObject, "BaseDoc"] class Comparison: @@ -23,20 +25,20 @@ class Comparison: def __init__( self, - actual: Union[JsonDict, YamlData, "BaseFormat"], - expected: Union[JsonDict, YamlData, "BaseFormat"], - format_class: Type["BaseFormat"], + actual: DictOrYamlObject, + expected: DictOrYamlObject, + format_class: Type["BaseDoc"], ) -> None: self.flat_actual = self._normalize_value(actual) self.flat_expected = self._normalize_value(expected) self.format_class = format_class - self.missing: Optional[BaseFormat] = None - self.missing_dict: Union[JsonDict, YamlData] = {} + self.missing: Optional[BaseDoc] = None + self.missing_dict: Union[JsonDict, YamlObject] = {} - self.diff: Optional[BaseFormat] = None - self.diff_dict: Union[JsonDict, YamlData] = {} + self.diff: Optional[BaseDoc] = None + self.diff_dict: Union[JsonDict, YamlObject] = {} @property def has_changes(self) -> bool: @@ -44,9 +46,9 @@ def has_changes(self) -> bool: return bool(self.missing or self.diff) @staticmethod - def _normalize_value(value: Union[JsonDict, YamlData, "BaseFormat"]) -> JsonDict: - if isinstance(value, BaseFormat): - dict_value: JsonDict = value.as_data + def _normalize_value(value: DictOrYamlObject) -> JsonDict: + if isinstance(value, BaseDoc): + dict_value: JsonDict = value.as_object else: dict_value = value return flatten(dict_value) @@ -57,7 +59,7 @@ def set_missing(self, missing_dict): return self.missing_dict = missing_dict if self.format_class: - self.missing = self.format_class(data=missing_dict) + self.missing = self.format_class(obj=missing_dict) def set_diff(self, diff_dict): """Set the diff dict and corresponding format.""" @@ -65,7 +67,7 @@ def set_diff(self, diff_dict): return self.diff_dict = diff_dict if self.format_class: - self.diff = self.format_class(data=diff_dict) + self.diff = self.format_class(obj=diff_dict) def update_pair(self, key, raw_expected): """Update a key on one of the comparison dicts, with its raw expected value.""" @@ -87,30 +89,25 @@ def update_pair(self, key, raw_expected): logger.warning("Err... this is unexpected, please open an issue: key={} raw_expected={}", key, raw_expected) -class BaseFormat(metaclass=abc.ABCMeta): +class BaseDoc(metaclass=abc.ABCMeta): """Base class for configuration file formats. :param path: Path of the config file to be loaded. :param string: Config in string format. - :param data: Config data in Python format (dict, YAMLFormat, TOMLFormat instances). + :param obj: Config object (Python dict, YamlDoc, TomlDoc instances). :param ignore_keys: List of keys to ignore when using the comparison methods. """ __repr__ = autorepr(["path"]) def __init__( - self, - *, - path: PathOrStr = None, - string: str = None, - data: Union[JsonDict, YamlData, "BaseFormat"] = None, - ignore_keys: List[str] = None + self, *, path: PathOrStr = None, string: str = None, obj: DictOrYamlObject = None, ignore_keys: List[str] = None ) -> None: self.path = path self._string = string - self._data = data - if path is None and string is None and data is None: - raise RuntimeError("Inform at least one argument: path, string or data") + self._object = obj + if path is None and string is None and obj is None: + raise RuntimeError("Inform at least one argument: path, string or object") self._ignore_keys = ignore_keys or [] self._reformatted: Optional[str] = None @@ -128,11 +125,11 @@ def as_string(self) -> str: return self._string or "" @property - def as_data(self) -> Union[JsonDict, YamlData]: - """String content converted to a Python data structure (a dict, YAML data, etc.).""" - if self._data is None: + def as_object(self) -> Union[JsonDict, YamlObject]: + """String content converted to a Python object (dict, YAML object instance, etc.).""" + if self._object is None: self.load() - return self._data or {} + return self._object or {} @property def reformatted(self) -> str: @@ -143,21 +140,21 @@ def reformatted(self) -> str: @classmethod def cleanup(cls, *args: List[Any]) -> List[Any]: - """Cleanup similar values according to the specific format. E.g.: YAMLFormat accepts 'True' or 'true'.""" + """Cleanup similar values according to the specific format. E.g.: YamlDoc accepts 'True' or 'true'.""" return list(*args) - def _create_comparison(self, expected: Union[JsonDict, YamlData, "BaseFormat"]): + def _create_comparison(self, expected: DictOrYamlObject): if not self._ignore_keys: - return Comparison(self.as_data or {}, expected or {}, self.__class__) + return Comparison(self.as_object or {}, expected or {}, self.__class__) - actual_original: Union[JsonDict, YamlData] = self.as_data or {} + actual_original: Union[JsonDict, YamlObject] = self.as_object or {} actual_copy = actual_original.copy() if isinstance(actual_original, dict) else actual_original - expected_original: Union[JsonDict, YamlData, "BaseFormat"] = expected or {} + expected_original: DictOrYamlObject = expected or {} if isinstance(expected_original, dict): expected_copy = expected_original.copy() - elif isinstance(expected_original, BaseFormat): - expected_copy = expected_original.as_data.copy() + elif isinstance(expected_original, BaseDoc): + expected_copy = expected_original.as_object.copy() else: expected_copy = expected_original for key in self._ignore_keys: @@ -165,7 +162,7 @@ def _create_comparison(self, expected: Union[JsonDict, YamlData, "BaseFormat"]): expected_copy.pop(key, None) return Comparison(actual_copy, expected_copy, self.__class__) - def compare_with_flatten(self, expected: Union[JsonDict, "BaseFormat"] = None) -> Comparison: + def compare_with_flatten(self, expected: Union[JsonDict, "BaseDoc"] = None) -> Comparison: """Compare two flattened dictionaries and compute missing and different items.""" comparison = self._create_comparison(expected) if comparison.flat_expected.items() <= comparison.flat_actual.items(): @@ -186,7 +183,7 @@ def compare_with_flatten(self, expected: Union[JsonDict, "BaseFormat"] = None) - return comparison def compare_with_dictdiffer( - self, expected: Union[JsonDict, "BaseFormat"] = None, transform_function: Callable = None + self, expected: Union[JsonDict, "BaseDoc"] = None, transform_function: Callable = None ) -> Comparison: """Compare two structures and compute missing and different items using ``dictdiffer``.""" comparison = self._create_comparison(expected) @@ -208,7 +205,21 @@ def compare_with_dictdiffer( return comparison -class TOMLFormat(BaseFormat): +class InlineTableTomlDecoder(toml.TomlDecoder): # type: ignore[name-defined] + """A hacky decoder to work around some bug (or unfinished work) in the Python TOML package. + + https://github.com/uiri/toml/issues/362. + """ + + def get_empty_inline_table(self): + """Hackity hack for a crappy unmaintained package. + + Total lack of respect, the guy doesn't even reply: https://github.com/uiri/toml/issues/361 + """ + return self.get_empty_table() + + +class TomlDoc(BaseDoc): """TOML configuration format.""" def load(self) -> bool: @@ -220,57 +231,90 @@ def load(self) -> bool: if self._string is not None: # TODO: I tried to replace toml by tomlkit, but lots of tests break. # The conversion to OrderedDict is not being done recursively (although I'm not sure this is a problem). - # self._data = OrderedDict(tomlkit.loads(self._string)) - self._data = toml.loads(self._string, _dict=OrderedDict) - if self._data is not None: - if isinstance(self._data, BaseFormat): - self._reformatted = self._data.reformatted + # self._object = OrderedDict(tomlkit.loads(self._string)) + self._object = toml.loads(self._string, decoder=InlineTableTomlDecoder(OrderedDict)) # type: ignore[call-arg] + if self._object is not None: + if isinstance(self._object, BaseDoc): + self._reformatted = self._object.reformatted else: # TODO: tomlkit.dumps() renders comments and I didn't find a way to turn this off, # but comments are being lost when the TOML plugin does dict comparisons. - # self._reformatted = tomlkit.dumps(OrderedDict(self._data), sort_keys=True) - self._reformatted = toml.dumps(self._data) + # self._reformatted = tomlkit.dumps(OrderedDict(self._object), sort_keys=True) + self._reformatted = toml.dumps(self._object) self._loaded = True return True -class YAMLFormat(BaseFormat): +def traverse_toml_tree(document: tomlkit.TOMLDocument, dictionary): + """Traverse a TOML document recursively and change values, keeping its formatting and comments.""" + for key, value in dictionary.items(): + if isinstance(value, (dict, OrderedDict)): + if key in document: + traverse_toml_tree(document[key], value) + else: + document.add(key, value) + else: + document[key] = value + + +class SensibleYAML(YAML): + """YAML with sensible defaults but an inefficient dump to string. + + `Output of dump() as a string + `_ + """ + + def __init__(self, *, typ=None, pure=False, output=None, plug_ins=None): + super().__init__(typ=typ, pure=pure, output=output, plug_ins=plug_ins) + self.map_indent = 2 + self.sequence_indent = 4 + self.sequence_dash_offset = 2 + + def loads(self, string: str): + """Load YAML from a string... that unusual use case in a world of files only.""" + return self.load(StringIO(string)) + + def dumps(self, data) -> str: + """Dump to a string... who would want such a thing? One can dump to a file or stdout.""" + output = StringIO() + self.dump(data, output, transform=None) + return output.getvalue() + + +class YamlDoc(BaseDoc): """YAML configuration format.""" + updater: SensibleYAML + def load(self) -> bool: """Load a YAML file by its path, a string or a dict.""" if self._loaded: return False - yaml = YAML() - yaml.map_indent = 2 - yaml.sequence_indent = 4 - yaml.sequence_dash_offset = 2 + self.updater = SensibleYAML() if self.path is not None: self._string = Path(self.path).read_text(encoding="UTF-8") if self._string is not None: - self._data = yaml.load(io.StringIO(self._string)) - if self._data is not None: - if isinstance(self._data, BaseFormat): - self._reformatted = self._data.reformatted + self._object = self.updater.loads(self._string) + if self._object is not None: + if isinstance(self._object, BaseDoc): + self._reformatted = self._object.reformatted else: - output = io.StringIO() - yaml.dump(self._data, output) - self._reformatted = output.getvalue() + self._reformatted = self.updater.dumps(self._object) self._loaded = True return True @property - def as_data(self) -> CommentedMap: + def as_object(self) -> CommentedMap: """On YAML, this dict is a special object with comments and ordered keys.""" - return super().as_data + return super().as_object @property def as_list(self) -> CommentedSeq: - """A list of dicts. On YAML, ``as_data`` might contain a ``list``. This property is just a proxy for typing.""" - return self.as_data + """A list of dicts, for typing purposes. On YAML, ``as_object`` might contain a ``list``.""" + return self.as_object @classmethod def cleanup(cls, *args: List[Any]) -> List[Any]: @@ -278,10 +322,53 @@ def cleanup(cls, *args: List[Any]) -> List[Any]: return [str(value).lower() if isinstance(value, (int, float, bool)) else value for value in args] -RoundTripRepresenter.add_representer(SortedDict, RoundTripRepresenter.represent_dict) +for dict_class in (SortedDict, OrderedDict): + RoundTripRepresenter.add_representer(dict_class, RoundTripRepresenter.represent_dict) + + +def is_scalar(value: YamlValue) -> bool: + """Return True if the value is NOT a dict or a list.""" + return not isinstance(value, (OrderedDict, list)) + + +def traverse_yaml_tree(yaml_obj: YamlObject, change: Union[JsonDict, OrderedDict]): + """Traverse a YAML document recursively and change values, keeping its formatting and comments.""" + for key, value in change.items(): + if key not in yaml_obj: + # Key doesn't exist: we can insert the whole nested OrderedDict at once, no regrets + last_pos = len(yaml_obj.keys()) + 1 + yaml_obj.insert(last_pos, key, value) + continue + + if is_scalar(value): + yaml_obj[key] = value + elif isinstance(value, OrderedDict): + traverse_yaml_tree(yaml_obj[key], value) + elif isinstance(value, list): + _traverse_yaml_list(yaml_obj, key, value) + + +def _traverse_yaml_list(yaml_obj: YamlObject, key: str, value: List[YamlValue]): + for index, element in enumerate(value): + insert: bool = index >= len(yaml_obj[key]) + + if not insert and is_scalar(yaml_obj[key][index]): + # If the original object is scalar, replace it with whatever element; + # without traversing, even if it's a dict + yaml_obj[key][index] = element + continue + + if is_scalar(element): + if insert: + yaml_obj[key].append(element) + else: + yaml_obj[key][index] = element + continue + + traverse_yaml_tree(yaml_obj[key][index], element) # type: ignore # mypy kept complaining about the Union -class JSONFormat(BaseFormat): +class JsonDoc(BaseDoc): """JSON configuration format.""" def load(self) -> bool: @@ -291,17 +378,17 @@ def load(self) -> bool: if self.path is not None: self._string = Path(self.path).read_text(encoding="UTF-8") if self._string is not None: - self._data = json.loads(self._string, object_pairs_hook=OrderedDict) - if self._data is not None: - if isinstance(self._data, BaseFormat): - self._reformatted = self._data.reformatted + self._object = json.loads(self._string, object_pairs_hook=OrderedDict) + if self._object is not None: + if isinstance(self._object, BaseDoc): + self._reformatted = self._object.reformatted else: # Every file should end with a blank line - self._reformatted = json.dumps(self._data, sort_keys=True, indent=2) + "\n" + self._reformatted = json.dumps(self._object, sort_keys=True, indent=2) + "\n" self._loaded = True return True @classmethod def cleanup(cls, *args: List[Any]) -> List[Any]: - """Cleanup similar values according to the specific format. E.g.: YAMLFormat accepts 'True' or 'true'.""" + """Cleanup similar values according to the specific format. E.g.: YamlDoc accepts 'True' or 'true'.""" return list(args) diff --git a/src/nitpick/fields.py b/src/nitpick/fields.py index 9f0755dd..b4ca2de8 100644 --- a/src/nitpick/fields.py +++ b/src/nitpick/fields.py @@ -37,7 +37,7 @@ def __init__(self, **kwargs): super().__init__(validate=validate, **kwargs) -class JSONString(fields.String): +class JsonString(fields.String): """A string field with valid JSON content.""" def __init__(self, **kwargs): diff --git a/src/nitpick/plugins/base.py b/src/nitpick/plugins/base.py index fdbb14c1..317d3289 100644 --- a/src/nitpick/plugins/base.py +++ b/src/nitpick/plugins/base.py @@ -2,14 +2,14 @@ import abc from functools import lru_cache from pathlib import Path -from typing import Iterator, Optional, Set +from typing import Iterator, Optional, Set, Type import jmespath from autorepr import autotext from loguru import logger from marshmallow import Schema -from nitpick.formats import Comparison +from nitpick.documents import BaseDoc, Comparison, DictOrYamlObject from nitpick.generic import search_dict from nitpick.plugins.info import FileInfo from nitpick.typedefs import JsonDict, mypy_property @@ -21,7 +21,7 @@ class NitpickPlugin(metaclass=abc.ABCMeta): :param data: File information (project, path, tags). :param expected_config: Expected configuration for the file - :param fix: Flag to modify files, if the plugin supports it (default: True). + :param autofix: Flag to modify files, if the plugin supports it (default: True). """ __str__, __unicode__ = autotext("{self.info.path_from_root} ({self.__class__.__name__})") @@ -29,11 +29,11 @@ class NitpickPlugin(metaclass=abc.ABCMeta): filename = "" # TODO: remove filename attribute after fixing dynamic/fixed schema loading violation_base_code: int = 0 - #: Can this plugin modify files directly? - can_fix: bool = False + #: Can this plugin modify its files directly? Are the files fixable? + fixable: bool = False #: Nested validation field for this file, to be applied in runtime when the validation schema is rebuilt. - #: Useful when you have a strict configuration for a file type (e.g. :py:class:`nitpick.plugins.json.JSONPlugin`). + #: Useful when you have a strict configuration for a file type (e.g. :py:class:`nitpick.plugins.json.JsonPlugin`). validation_schema: Optional[Schema] = None #: Which ``identify`` tags this :py:class:`nitpick.plugins.base.NitpickPlugin` child recognises. @@ -41,7 +41,7 @@ class NitpickPlugin(metaclass=abc.ABCMeta): skip_empty_suggestion = False - def __init__(self, info: FileInfo, expected_config: JsonDict, fix=False) -> None: + def __init__(self, info: FileInfo, expected_config: JsonDict, autofix=False) -> None: self.info = info self.filename = info.path_from_root self.reporter = Reporter(info, self.violation_base_code) @@ -51,7 +51,7 @@ def __init__(self, info: FileInfo, expected_config: JsonDict, fix=False) -> None # Configuration for this file as a TOML dict, taken from the style file. self.expected_config: JsonDict = expected_config or {} - self.fix = self.can_fix and fix + self.autofix = self.fixable and autofix # Dirty flag to avoid changing files without need self.dirty: bool = False @@ -91,7 +91,7 @@ def _enforce_file_configuration(self): else: yield from self._suggest_when_file_not_found() - if self.fix and self.dirty: + if self.autofix and self.dirty: fuss = self.write_file(file_exists) # pylint: disable=assignment-from-none if fuss: yield fuss @@ -110,12 +110,12 @@ def _suggest_when_file_not_found(self): logger.info(f"{self}: Suggest initial contents for {self.filename}") if suggestion: - yield self.reporter.make_fuss(SharedViolations.CREATE_FILE_WITH_SUGGESTION, suggestion, fixed=self.fix) + yield self.reporter.make_fuss(SharedViolations.CREATE_FILE_WITH_SUGGESTION, suggestion, fixed=self.autofix) else: yield self.reporter.make_fuss(SharedViolations.CREATE_FILE) def write_file(self, file_exists: bool) -> Optional[Fuss]: # pylint: disable=unused-argument,no-self-use - """Hook to write the new file when fix mode is on. Should be used by inherited classes.""" + """Hook to write the new file when autofix mode is on. Should be used by inherited classes.""" return None @abc.abstractmethod @@ -127,6 +127,17 @@ def enforce_rules(self) -> Iterator[Fuss]: def initial_contents(self) -> str: """Suggested initial content when the file doesn't exist.""" + def write_initial_contents(self, format_class: Type[BaseDoc], expected_obj: DictOrYamlObject = None) -> str: + """Helper to write initial contents based on a format.""" + if not expected_obj: + expected_obj = self.expected_config + + formatted_str = format_class(obj=expected_obj).reformatted + if self.autofix: + self.file_path.parent.mkdir(exist_ok=True, parents=True) + self.file_path.write_text(formatted_str) + return formatted_str + def warn_missing_different(self, comparison: Comparison, prefix: str = "") -> Iterator[Fuss]: """Warn about missing and different keys.""" # pylint: disable=not-callable diff --git a/src/nitpick/plugins/ini.py b/src/nitpick/plugins/ini.py index 6341f600..7424d6e8 100644 --- a/src/nitpick/plugins/ini.py +++ b/src/nitpick/plugins/ini.py @@ -31,7 +31,7 @@ class Violations(ViolationEnum): class IniPlugin(NitpickPlugin): - """Enforce config on INI files. + """Enforce configurations and autofix INI files. Examples of ``.ini`` files handled by this plugin: @@ -43,7 +43,7 @@ class IniPlugin(NitpickPlugin): Style examples enforcing values on INI files: :ref:`flake8 configuration `. """ - can_fix = True + fixable = True identify_tags = {"ini", "editorconfig"} violation_base_code = 320 @@ -122,7 +122,7 @@ def get_missing_output(self) -> str: parser = ConfigParser() for section in sorted(missing, key=lambda s: "0" if s == TOP_SECTION else f"1{s}"): expected_config: Dict = self.expected_config[section] - if self.fix: + if self.autofix: if self.updater.last_block: self.updater.last_block.add_after.space(1) self.updater.add_section(section) @@ -173,7 +173,7 @@ def _read_file(self) -> Iterator[Fuss]: return # Don't change the file if there was a parsing error - self.fix = False + self.autofix = False yield self.reporter.make_fuss(Violations.PARSING_ERROR, cls=parsing_err.__class__.__name__, msg=parsing_err) raise Error @@ -181,13 +181,13 @@ def enforce_missing_sections(self) -> Iterator[Fuss]: """Enforce missing sections.""" missing = self.get_missing_output() if missing: - yield self.reporter.make_fuss(Violations.MISSING_SECTIONS, missing, self.fix) + yield self.reporter.make_fuss(Violations.MISSING_SECTIONS, missing, self.autofix) def enforce_section(self, section: str) -> Iterator[Fuss]: """Enforce rules for a section.""" expected_dict = self.expected_config[section] actual_dict = {k: v.value for k, v in self.updater[section].items()} - # TODO: add a class Ini(BaseFormat) and move this dictdiffer code there + # TODO: add a class Ini(BaseDoc) and move this dictdiffer code there for diff_type, key, values in dictdiffer.diff(actual_dict, expected_dict): if diff_type == dictdiffer.CHANGE: if f"{section}.{key}" in self.comma_separated_values: @@ -207,7 +207,7 @@ def enforce_comma_separated_values(self, section, key, raw_actual: Any, raw_expe joined_values = ",".join(sorted(missing)) value_to_append = f",{joined_values}" - if self.fix: + if self.autofix: self.updater[section][key].value += value_to_append self.dirty = True section_header = "" if section == TOP_SECTION else f"[{section}]\n" @@ -216,7 +216,7 @@ def enforce_comma_separated_values(self, section, key, raw_actual: Any, raw_expe Violations.MISSING_VALUES_IN_LIST, f"{section_header}{key} = (...){value_to_append}", key=key, - fixed=self.fix, + fixed=self.autofix, ) def compare_different_keys(self, section, key, raw_actual: Any, raw_expected: Any) -> Iterator[Fuss]: @@ -231,7 +231,7 @@ def compare_different_keys(self, section, key, raw_actual: Any, raw_expected: An if actual == expected: return - if self.fix: + if self.autofix: self.updater[section][key].value = expected self.dirty = True if section == TOP_SECTION: @@ -240,7 +240,7 @@ def compare_different_keys(self, section, key, raw_actual: Any, raw_expected: An f"{key} = {raw_expected}", key=key, actual=raw_actual, - fixed=self.fix, + fixed=self.autofix, ) else: yield self.reporter.make_fuss( @@ -249,7 +249,7 @@ def compare_different_keys(self, section, key, raw_actual: Any, raw_expected: An section=section, key=key, actual=raw_actual, - fixed=self.fix, + fixed=self.autofix, ) def show_missing_keys(self, section: str, values: List[Tuple[str, Any]]) -> Iterator[Fuss]: @@ -262,14 +262,14 @@ def show_missing_keys(self, section: str, values: List[Tuple[str, Any]]) -> Iter if section == TOP_SECTION: yield self.reporter.make_fuss( - Violations.TOP_SECTION_MISSING_OPTION, self.contents_without_top_section(output), self.fix + Violations.TOP_SECTION_MISSING_OPTION, self.contents_without_top_section(output), self.autofix ) else: - yield self.reporter.make_fuss(Violations.MISSING_OPTION, output, self.fix, section=section) + yield self.reporter.make_fuss(Violations.MISSING_OPTION, output, self.autofix, section=section) def add_options_before_space(self, section: str, options: OrderedDict) -> None: """Add new options before a blank line in the end of the section.""" - if not self.fix: + if not self.autofix: return space_removed = False diff --git a/src/nitpick/plugins/json.py b/src/nitpick/plugins/json.py index b6da6415..71fed215 100644 --- a/src/nitpick/plugins/json.py +++ b/src/nitpick/plugins/json.py @@ -6,7 +6,7 @@ from loguru import logger from nitpick import fields -from nitpick.formats import BaseFormat, JSONFormat +from nitpick.documents import BaseDoc, JsonDoc from nitpick.generic import DictBlender, flatten, unflatten from nitpick.plugins import hookimpl from nitpick.plugins.base import NitpickPlugin @@ -20,29 +20,29 @@ VALUE_PLACEHOLDER = "" -class JSONFileSchema(BaseNitpickSchema): +class JsonFileSchema(BaseNitpickSchema): """Validation schema for any JSON file added to the style.""" contains_keys = fields.List(fields.NonEmptyString) - contains_json = fields.Dict(fields.NonEmptyString, fields.JSONString) + contains_json = fields.Dict(fields.NonEmptyString, fields.JsonString) -class JSONPlugin(NitpickPlugin): - """Enforce configurations for any JSON file. +class JsonPlugin(NitpickPlugin): + """Enforce configurations and autofix JSON files. Add the configurations for the file name you wish to check. Style example: :ref:`the default config for package.json `. """ - validation_schema = JSONFileSchema + validation_schema = JsonFileSchema identify_tags = {"json"} violation_base_code = 340 - can_fix = True + fixable = True def enforce_rules(self) -> Iterator[Fuss]: """Enforce rules for missing keys and JSON content.""" - actual = JSONFormat(path=self.file_path) - final_dict: Optional[JsonDict] = flatten(actual.as_data) if self.fix else None + actual = JsonDoc(path=self.file_path) + final_dict: Optional[JsonDict] = flatten(actual.as_object) if self.autofix else None comparison = actual.compare_with_flatten(self.expected_dict_from_contains_keys()) if comparison.missing: @@ -55,8 +55,8 @@ def enforce_rules(self) -> Iterator[Fuss]: self.report(SharedViolations.MISSING_VALUES, final_dict, comparison.missing), ) - if self.fix and self.dirty and final_dict: - self.file_path.write_text(JSONFormat(data=unflatten(final_dict)).reformatted) + if self.autofix and self.dirty and final_dict: + self.file_path.write_text(JsonDoc(obj=unflatten(final_dict)).reformatted) def expected_dict_from_contains_keys(self): """Expected dict created from "contains_keys" values.""" @@ -76,35 +76,32 @@ def expected_dict_from_contains_json(self): continue return expected_config - def report(self, violation: ViolationEnum, final_dict: Optional[JsonDict], change: Optional[BaseFormat]): + def report(self, violation: ViolationEnum, final_dict: Optional[JsonDict], change: Optional[BaseDoc]): """Report a violation while optionally modifying the JSON dict.""" if not change: return if final_dict: - final_dict.update(flatten(change.as_data)) + final_dict.update(flatten(change.as_object)) self.dirty = True - yield self.reporter.make_fuss(violation, change.reformatted, prefix="", fixed=self.fix) + yield self.reporter.make_fuss(violation, change.reformatted, prefix="", fixed=self.autofix) @property def initial_contents(self) -> str: """Suggest the initial content for this missing file.""" suggestion = DictBlender(self.expected_dict_from_contains_keys()) suggestion.add(self.expected_dict_from_contains_json()) - json_as_string = JSONFormat(data=suggestion.mix()).reformatted if suggestion else "" - if self.fix: - self.file_path.write_text(json_as_string) - return json_as_string + return self.write_initial_contents(JsonDoc, suggestion.mix()) @hookimpl def plugin_class() -> Type["NitpickPlugin"]: """Handle JSON files.""" - return JSONPlugin + return JsonPlugin @hookimpl def can_handle(info: FileInfo) -> Optional[Type["NitpickPlugin"]]: """Handle JSON files.""" - if JSONPlugin.identify_tags & info.tags: - return JSONPlugin + if JsonPlugin.identify_tags & info.tags: + return JsonPlugin return None diff --git a/src/nitpick/plugins/pre_commit.py b/src/nitpick/plugins/pre_commit.py index c0f74b17..57072fba 100644 --- a/src/nitpick/plugins/pre_commit.py +++ b/src/nitpick/plugins/pre_commit.py @@ -5,12 +5,12 @@ import attr from nitpick.constants import PRE_COMMIT_CONFIG_YAML -from nitpick.formats import YAMLFormat +from nitpick.documents import YamlDoc from nitpick.generic import find_object_by_key, search_dict from nitpick.plugins import hookimpl from nitpick.plugins.base import NitpickPlugin from nitpick.plugins.info import FileInfo -from nitpick.typedefs import JsonDict, YamlData +from nitpick.typedefs import JsonDict, YamlObject from nitpick.violations import Fuss, ViolationEnum KEY_REPOS = "repos" @@ -26,7 +26,7 @@ class PreCommitHook: repo = attr.ib(type=str) hook_id = attr.ib(type=str) - yaml = attr.ib(type=YAMLFormat) + yaml = attr.ib(type=YamlDoc) @property def unique_key(self) -> str: @@ -44,9 +44,9 @@ def single_hook(self) -> JsonDict: return self.yaml.as_list[0] @classmethod - def get_all_hooks_from(cls, str_or_yaml: Union[str, YamlData]): + def get_all_hooks_from(cls, str_or_yaml: Union[str, YamlObject]): """Get all hooks from a YAML string. Split the string in hooks and copy the repo info for each.""" - yaml = YAMLFormat(string=str_or_yaml).as_list if isinstance(str_or_yaml, str) else str_or_yaml + yaml = YamlDoc(string=str_or_yaml).as_list if isinstance(str_or_yaml, str) else str_or_yaml hooks = [] for repo in yaml: for index, hook in enumerate(repo.get(KEY_HOOKS, [])): @@ -55,7 +55,7 @@ def get_all_hooks_from(cls, str_or_yaml: Union[str, YamlData]): hook_data_only = search_dict(f"{KEY_HOOKS}[{index}]", repo, {}) repo_data_only.update({KEY_HOOKS: [hook_data_only]}) hooks.append( - PreCommitHook(repo.get(KEY_REPO), hook[KEY_ID], YAMLFormat(data=[repo_data_only])).key_value_pair + PreCommitHook(repo.get(KEY_REPO), hook[KEY_ID], YamlDoc(obj=[repo_data_only])).key_value_pair ) return OrderedDict(hooks) @@ -82,7 +82,7 @@ class PreCommitPlugin(NitpickPlugin): filename = PRE_COMMIT_CONFIG_YAML violation_base_code = 330 - actual_yaml: YAMLFormat + actual_yaml: YamlDoc actual_hooks: Dict[str, PreCommitHook] = OrderedDict() actual_hooks_by_key: Dict[str, int] = {} actual_hooks_by_index: List[str] = [] @@ -96,15 +96,15 @@ def initial_contents(self) -> str: for repo in original_repos: if KEY_YAML not in repo: continue - repo_list = YAMLFormat(string=repo[KEY_YAML]).as_list + repo_list = YamlDoc(string=repo[KEY_YAML]).as_list suggested[KEY_REPOS].extend(repo_list) suggested.update(original) - return YAMLFormat(data=suggested).reformatted + return YamlDoc(obj=suggested).reformatted def enforce_rules(self) -> Iterator[Fuss]: """Enforce rules for the pre-commit hooks.""" - self.actual_yaml = YAMLFormat(path=self.file_path) - if KEY_REPOS not in self.actual_yaml.as_data: + self.actual_yaml = YamlDoc(path=self.file_path) + if KEY_REPOS not in self.actual_yaml.as_object: # TODO: if the 'repos' key doesn't exist, assume repos are in the root of the .yml file # Having the 'repos' key is not actually a requirement. 'pre-commit-validate-config' works without it. yield self.reporter.make_fuss(Violations.NO_ROOT_KEY) @@ -112,7 +112,7 @@ def enforce_rules(self) -> Iterator[Fuss]: # Check the root values in the configuration file yield from self.warn_missing_different( - YAMLFormat(data=self.actual_yaml.as_data, ignore_keys=[KEY_REPOS]).compare_with_dictdiffer( + YamlDoc(obj=self.actual_yaml.as_object, ignore_keys=[KEY_REPOS]).compare_with_dictdiffer( self.expected_config ) ) @@ -121,7 +121,7 @@ def enforce_rules(self) -> Iterator[Fuss]: def enforce_hooks(self) -> Iterator[Fuss]: """Enforce the repositories configured in pre-commit.""" - self.actual_hooks = PreCommitHook.get_all_hooks_from(self.actual_yaml.as_data.get(KEY_REPOS)) + self.actual_hooks = PreCommitHook.get_all_hooks_from(self.actual_yaml.as_object.get(KEY_REPOS)) self.actual_hooks_by_key = {name: index for index, name in enumerate(self.actual_hooks)} self.actual_hooks_by_index = list(self.actual_hooks) @@ -135,15 +135,15 @@ def enforce_hooks(self) -> Iterator[Fuss]: def enforce_repo_block(self, expected_repo_block: OrderedDict) -> Iterator[Fuss]: """Enforce a repo with a YAML string configuration.""" - expected_hooks = PreCommitHook.get_all_hooks_from(YAMLFormat(string=expected_repo_block.get(KEY_YAML)).as_list) + expected_hooks = PreCommitHook.get_all_hooks_from(YamlDoc(string=expected_repo_block.get(KEY_YAML)).as_list) for unique_key, hook in expected_hooks.items(): if unique_key not in self.actual_hooks: yield self.reporter.make_fuss( - Violations.HOOK_NOT_FOUND, YAMLFormat(data=hook.yaml.as_data).reformatted, id=hook.hook_id + Violations.HOOK_NOT_FOUND, YamlDoc(obj=hook.yaml.as_object).reformatted, id=hook.hook_id ) continue - comparison = YAMLFormat(data=self.actual_hooks[unique_key].single_hook).compare_with_dictdiffer( + comparison = YamlDoc(obj=self.actual_hooks[unique_key].single_hook).compare_with_dictdiffer( hook.single_hook ) @@ -154,7 +154,7 @@ def enforce_repo_block(self, expected_repo_block: OrderedDict) -> Iterator[Fuss] def enforce_repo_old_format(self, index: int, repo_data: OrderedDict) -> Iterator[Fuss]: """Enforce repos using the old deprecated format with ``hooks`` and ``repo`` keys.""" - actual: List[YamlData] = self.actual_yaml.as_data.get(KEY_REPOS, []) + actual: List[YamlObject] = self.actual_yaml.as_object.get(KEY_REPOS, []) repo_name = repo_data.get(KEY_REPO) @@ -176,7 +176,7 @@ def enforce_repo_old_format(self, index: int, repo_data: OrderedDict) -> Iterato yield self.reporter.make_fuss(Violations.STYLE_FILE_MISSING_NAME, key=KEY_HOOKS, repo=repo_name) return - expected_hooks = YAMLFormat(string=yaml_expected_hooks).as_data + expected_hooks = YamlDoc(string=yaml_expected_hooks).as_object for expected_dict in expected_hooks: hook_id = expected_dict.get(KEY_ID) expected_yaml = self.format_hook(expected_dict).rstrip() @@ -189,7 +189,7 @@ def enforce_repo_old_format(self, index: int, repo_data: OrderedDict) -> Iterato @staticmethod def format_hook(expected_dict) -> str: """Format the hook so it's easy to copy and paste it to the .yaml file: ID goes first, indent with spaces.""" - lines = YAMLFormat(data=expected_dict).reformatted + lines = YamlDoc(obj=expected_dict).reformatted output: List[str] = [] for line in lines.split("\n"): if line.startswith("id:"): @@ -208,6 +208,7 @@ def plugin_class() -> Type["NitpickPlugin"]: @hookimpl def can_handle(info: FileInfo) -> Optional[Type["NitpickPlugin"]]: """Handle pre-commit config file.""" + # TODO: the YAML plugin should handle this file, now or later if info.path_from_root == PRE_COMMIT_CONFIG_YAML: return PreCommitPlugin return None diff --git a/src/nitpick/plugins/text.py b/src/nitpick/plugins/text.py index fd996135..98eeb598 100644 --- a/src/nitpick/plugins/text.py +++ b/src/nitpick/plugins/text.py @@ -12,6 +12,7 @@ from nitpick.violations import Fuss, ViolationEnum TEXT_FILE_RTFD_PAGE = "plugins.html#text-files" +KEY_CONTAINS = "contains" class TextItemSchema(Schema): @@ -58,7 +59,7 @@ class TextPlugin(NitpickPlugin): violation_base_code = 350 def _expected_lines(self): - return [obj.get("line") for obj in self.expected_config.get("contains", {})] + return [obj.get("line") for obj in self.expected_config.get(KEY_CONTAINS, {})] @property def initial_contents(self) -> str: diff --git a/src/nitpick/plugins/toml.py b/src/nitpick/plugins/toml.py index 37c96a15..cb23cb8a 100644 --- a/src/nitpick/plugins/toml.py +++ b/src/nitpick/plugins/toml.py @@ -1,32 +1,19 @@ """TOML files.""" -from collections import OrderedDict from itertools import chain from typing import Iterator, Optional, Type from tomlkit import dumps, parse from tomlkit.toml_document import TOMLDocument -from nitpick.formats import BaseFormat, TOMLFormat +from nitpick.documents import BaseDoc, TomlDoc, traverse_toml_tree from nitpick.plugins import hookimpl from nitpick.plugins.base import NitpickPlugin from nitpick.plugins.info import FileInfo from nitpick.violations import Fuss, SharedViolations, ViolationEnum -def change_toml(document: TOMLDocument, dictionary): - """Traverse a TOML document recursively and change values, keeping its formatting and comments.""" - for key, value in dictionary.items(): - if isinstance(value, (dict, OrderedDict)): - if key in document: - change_toml(document[key], value) - else: - document.add(key, value) - else: - document[key] = value - - class TomlPlugin(NitpickPlugin): - """Enforce config on TOML files. + """Enforce configurations and autofix TOML files. E.g.: `pyproject.toml (PEP 518) `_. @@ -39,39 +26,36 @@ class TomlPlugin(NitpickPlugin): identify_tags = {"toml"} violation_base_code = 310 - can_fix = True + fixable = True def enforce_rules(self) -> Iterator[Fuss]: """Enforce rules for missing key/value pairs in the TOML file.""" - toml_format = TOMLFormat(path=self.file_path) - comparison = toml_format.compare_with_flatten(self.expected_config) + toml_doc = TomlDoc(path=self.file_path) + comparison = toml_doc.compare_with_flatten(self.expected_config) if not comparison.has_changes: return - document = parse(toml_format.as_string) if self.fix else None + document = parse(toml_doc.as_string) if self.autofix else None yield from chain( self.report(SharedViolations.DIFFERENT_VALUES, document, comparison.diff), self.report(SharedViolations.MISSING_VALUES, document, comparison.missing), ) - if self.fix and self.dirty: + if self.autofix and self.dirty: self.file_path.write_text(dumps(document)) - def report(self, violation: ViolationEnum, document: Optional[TOMLDocument], change: Optional[BaseFormat]): + def report(self, violation: ViolationEnum, document: Optional[TOMLDocument], change: Optional[BaseDoc]): """Report a violation while optionally modifying the TOML document.""" if not change: return if document: - change_toml(document, change.as_data) + traverse_toml_tree(document, change.as_object) self.dirty = True - yield self.reporter.make_fuss(violation, change.reformatted.strip(), prefix="", fixed=self.fix) + yield self.reporter.make_fuss(violation, change.reformatted.strip(), prefix="", fixed=self.autofix) @property def initial_contents(self) -> str: """Suggest the initial content for this missing file.""" - toml_as_string = TOMLFormat(data=self.expected_config).reformatted - if self.fix: - self.file_path.write_text(toml_as_string) - return toml_as_string + return self.write_initial_contents(TomlDoc) @hookimpl diff --git a/src/nitpick/plugins/yaml.py b/src/nitpick/plugins/yaml.py new file mode 100644 index 00000000..9ede4b71 --- /dev/null +++ b/src/nitpick/plugins/yaml.py @@ -0,0 +1,71 @@ +"""YAML files.""" +from itertools import chain +from typing import Iterator, Optional, Type + +from nitpick.constants import PRE_COMMIT_CONFIG_YAML +from nitpick.documents import BaseDoc, YamlDoc, traverse_yaml_tree +from nitpick.plugins import hookimpl +from nitpick.plugins.base import NitpickPlugin +from nitpick.plugins.info import FileInfo +from nitpick.plugins.text import KEY_CONTAINS +from nitpick.typedefs import YamlObject +from nitpick.violations import Fuss, SharedViolations, ViolationEnum + + +class YamlPlugin(NitpickPlugin): + """Enforce configurations and autofix YAML files.""" + + identify_tags = {"yaml"} + violation_base_code = 360 + fixable = True + + def enforce_rules(self) -> Iterator[Fuss]: + """Enforce rules for missing data in the YAML file.""" + if KEY_CONTAINS in self.expected_config: + # If the expected configuration has this key, it means that this config is being handled by TextPlugin. + # TODO: A YAML file that has a "contains" key on its root cannot be handled as YAML... how to fix this? + return + + yaml_doc = YamlDoc(path=self.file_path) + comparison = yaml_doc.compare_with_flatten(self.expected_config) + if not comparison.has_changes: + return + + yield from chain( + self.report(SharedViolations.DIFFERENT_VALUES, yaml_doc.as_object, comparison.diff), + self.report(SharedViolations.MISSING_VALUES, yaml_doc.as_object, comparison.missing), + ) + if self.autofix and self.dirty: + yaml_doc.updater.dump(yaml_doc.as_object, self.file_path) + + def report(self, violation: ViolationEnum, yaml_object: YamlObject, change: Optional[BaseDoc]): + """Report a violation while optionally modifying the YAML document.""" + if not change: + return + if self.autofix: + traverse_yaml_tree(yaml_object, change.as_object) + self.dirty = True + yield self.reporter.make_fuss(violation, change.reformatted.strip(), prefix="", fixed=self.autofix) + + @property + def initial_contents(self) -> str: + """Suggest the initial content for this missing file.""" + return self.write_initial_contents(YamlDoc) + + +@hookimpl +def plugin_class() -> Type["NitpickPlugin"]: + """Handle YAML files.""" + return YamlPlugin + + +@hookimpl +def can_handle(info: FileInfo) -> Optional[Type["NitpickPlugin"]]: + """Handle YAML files.""" + if info.path_from_root == PRE_COMMIT_CONFIG_YAML: + # TODO: For now, this plugin won't touch the current pre-commit config + return None + + if YamlPlugin.identify_tags & info.tags: + return YamlPlugin + return None diff --git a/src/nitpick/project.py b/src/nitpick/project.py index 416f88f6..07838749 100644 --- a/src/nitpick/project.py +++ b/src/nitpick/project.py @@ -30,8 +30,8 @@ TOOL_NITPICK_JMEX, TOOL_NITPICK_KEY, ) +from nitpick.documents import TomlDoc from nitpick.exceptions import QuitComplainingError -from nitpick.formats import TOMLFormat from nitpick.generic import search_dict, version_to_tuple from nitpick.schemas import BaseNitpickSchema, flatten_marshmallow_errors, help_message from nitpick.typedefs import JsonDict, PathOrStr, mypy_property @@ -148,8 +148,8 @@ def read_configuration(self) -> Configuration: logger.warning("Config file: none found") return Configuration(None, [], "") - toml_format = TOMLFormat(path=config_file) - config_dict = search_dict(TOOL_NITPICK_JMEX, toml_format.as_data, {}) + toml_doc = TomlDoc(path=config_file) + config_dict = search_dict(TOOL_NITPICK_JMEX, toml_doc.as_object, {}) validation_errors = ToolNitpickSectionSchema().validate(config_dict) if not validation_errors: return Configuration(config_file, config_dict.get("style", []), config_dict.get("cache", "")) diff --git a/src/nitpick/resources/__init__.py b/src/nitpick/resources/__init__.py new file mode 100644 index 00000000..f8dfdd02 --- /dev/null +++ b/src/nitpick/resources/__init__.py @@ -0,0 +1 @@ +"""A library of Nitpick styles. Building blocks that can be combined and reused.""" diff --git a/src/nitpick/resources/any/__init__.py b/src/nitpick/resources/any/__init__.py new file mode 100644 index 00000000..65a33e80 --- /dev/null +++ b/src/nitpick/resources/any/__init__.py @@ -0,0 +1 @@ +"""Styles for any language.""" diff --git a/styles/commitizen.toml b/src/nitpick/resources/any/commitizen.toml similarity index 100% rename from styles/commitizen.toml rename to src/nitpick/resources/any/commitizen.toml diff --git a/src/nitpick/resources/any/commitlint.toml b/src/nitpick/resources/any/commitlint.toml new file mode 100644 index 00000000..0e86466a --- /dev/null +++ b/src/nitpick/resources/any/commitlint.toml @@ -0,0 +1,17 @@ +["package.json".contains_json] +commitlint = """ + { + "extends": [ + "@commitlint/config-conventional" + ] + } +""" + +[[".pre-commit-config.yaml".repos]] +yaml = """ + - repo: https://github.com/alessandrojcm/commitlint-pre-commit-hook + hooks: + - id: commitlint + stages: [commit-msg] + additional_dependencies: ['@commitlint/config-conventional'] +""" diff --git a/styles/editorconfig.toml b/src/nitpick/resources/any/editorconfig.toml similarity index 100% rename from styles/editorconfig.toml rename to src/nitpick/resources/any/editorconfig.toml diff --git a/styles/pre-commit/main.toml b/src/nitpick/resources/any/hooks.toml similarity index 55% rename from styles/pre-commit/main.toml rename to src/nitpick/resources/any/hooks.toml index ae36bc71..d8d213f4 100644 --- a/styles/pre-commit/main.toml +++ b/src/nitpick/resources/any/hooks.toml @@ -3,3 +3,11 @@ [nitpick.files.present] ".pre-commit-config.yaml" = "Create the file with the contents below, then run 'pre-commit install'" + +[[".pre-commit-config.yaml".repos]] +yaml = """ + - repo: https://github.com/pre-commit/pre-commit-hooks + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace +""" diff --git a/src/nitpick/resources/javascript/__init__.py b/src/nitpick/resources/javascript/__init__.py new file mode 100644 index 00000000..cd61c349 --- /dev/null +++ b/src/nitpick/resources/javascript/__init__.py @@ -0,0 +1 @@ +"""Styles for JavaScript.""" diff --git a/styles/package-json.toml b/src/nitpick/resources/javascript/package-json.toml similarity index 100% rename from styles/package-json.toml rename to src/nitpick/resources/javascript/package-json.toml diff --git a/src/nitpick/resources/python/__init__.py b/src/nitpick/resources/python/__init__.py new file mode 100644 index 00000000..bb0e97d9 --- /dev/null +++ b/src/nitpick/resources/python/__init__.py @@ -0,0 +1 @@ +"""Styles for Python.""" diff --git a/styles/absent-files.toml b/src/nitpick/resources/python/absent.toml similarity index 100% rename from styles/absent-files.toml rename to src/nitpick/resources/python/absent.toml diff --git a/styles/black.toml b/src/nitpick/resources/python/black.toml similarity index 100% rename from styles/black.toml rename to src/nitpick/resources/python/black.toml diff --git a/styles/flake8.toml b/src/nitpick/resources/python/flake8.toml similarity index 100% rename from styles/flake8.toml rename to src/nitpick/resources/python/flake8.toml diff --git a/styles/pre-commit/python.toml b/src/nitpick/resources/python/hooks.toml similarity index 100% rename from styles/pre-commit/python.toml rename to src/nitpick/resources/python/hooks.toml diff --git a/styles/ipython.toml b/src/nitpick/resources/python/ipython.toml similarity index 100% rename from styles/ipython.toml rename to src/nitpick/resources/python/ipython.toml diff --git a/styles/isort.toml b/src/nitpick/resources/python/isort.toml similarity index 100% rename from styles/isort.toml rename to src/nitpick/resources/python/isort.toml diff --git a/styles/mypy.toml b/src/nitpick/resources/python/mypy.toml similarity index 66% rename from styles/mypy.toml rename to src/nitpick/resources/python/mypy.toml index fff23874..c608e222 100644 --- a/styles/mypy.toml +++ b/src/nitpick/resources/python/mypy.toml @@ -2,8 +2,8 @@ ["setup.cfg".mypy] ignore_missing_imports = true -# Do not follow imports (except for ones found in typeshed) -follow_imports = "skip" +# https://mypy.readthedocs.io/en/stable/running_mypy.html#follow-imports +follow_imports = "normal" # Treat Optional per PEP 484 strict_optional = true @@ -13,7 +13,8 @@ warn_no_return = true # Lint-style cleanliness for typing warn_redundant_casts = true -warn_unused_ignores = true +# False positives when running on local machine... it works on pre-commit.ci ¯\_(ツ)_/¯ +warn_unused_ignores = false [[".pre-commit-config.yaml".repos]] yaml = """ diff --git a/styles/poetry.toml b/src/nitpick/resources/python/poetry.toml similarity index 100% rename from styles/poetry.toml rename to src/nitpick/resources/python/poetry.toml diff --git a/styles/pylint.toml b/src/nitpick/resources/python/pylint.toml similarity index 96% rename from styles/pylint.toml rename to src/nitpick/resources/python/pylint.toml index cee76504..74ec1b28 100644 --- a/styles/pylint.toml +++ b/src/nitpick/resources/python/pylint.toml @@ -20,7 +20,7 @@ output-format = "colorized" # comma_separated_values = ["MESSAGES CONTROL.disable"] # This syntax will be deprecated anyway, so I won't make it work now # Configurations for the black formatter -#disable = "bad-continuation,bad-whitespace,fixme,cyclic-import" +#disable = "bad-continuation,bad-whitespace,fixme,cyclic-import,line-too-long" [".pylintrc".BASIC] # List of builtins function names that should not be used, separated by a comma diff --git a/styles/python310.toml b/src/nitpick/resources/python/python310.toml similarity index 100% rename from styles/python310.toml rename to src/nitpick/resources/python/python310.toml diff --git a/styles/python36.toml b/src/nitpick/resources/python/python36.toml similarity index 100% rename from styles/python36.toml rename to src/nitpick/resources/python/python36.toml diff --git a/styles/python37.toml b/src/nitpick/resources/python/python37.toml similarity index 100% rename from styles/python37.toml rename to src/nitpick/resources/python/python37.toml diff --git a/styles/python38.toml b/src/nitpick/resources/python/python38.toml similarity index 100% rename from styles/python38.toml rename to src/nitpick/resources/python/python38.toml diff --git a/styles/python39.toml b/src/nitpick/resources/python/python39.toml similarity index 100% rename from styles/python39.toml rename to src/nitpick/resources/python/python39.toml diff --git a/styles/tox.toml b/src/nitpick/resources/python/tox.toml similarity index 100% rename from styles/tox.toml rename to src/nitpick/resources/python/tox.toml diff --git a/src/nitpick/resources/shell/__init__.py b/src/nitpick/resources/shell/__init__.py new file mode 100644 index 00000000..cceb42ae --- /dev/null +++ b/src/nitpick/resources/shell/__init__.py @@ -0,0 +1 @@ +"""Styles for shell scripts.""" diff --git a/styles/pre-commit/bash.toml b/src/nitpick/resources/shell/hooks.toml similarity index 100% rename from styles/pre-commit/bash.toml rename to src/nitpick/resources/shell/hooks.toml diff --git a/src/nitpick/style/core.py b/src/nitpick/style/core.py index bec0deaa..50a15259 100644 --- a/src/nitpick/style/core.py +++ b/src/nitpick/style/core.py @@ -25,8 +25,8 @@ PYPROJECT_TOML, TOML_EXTENSION, ) +from nitpick.documents import TomlDoc from nitpick.exceptions import QuitComplainingError, pretty_exception -from nitpick.formats import TOMLFormat from nitpick.generic import DictBlender, is_url, search_dict from nitpick.plugins.base import NitpickPlugin from nitpick.plugins.info import FileInfo @@ -133,9 +133,9 @@ def _include_style(self, style_uri): yield from self.include_multiple_styles(sub_styles) def _read_toml(self, file_contents, style_path): - toml = TOMLFormat(string=file_contents) + toml = TomlDoc(string=file_contents) try: - read_toml_dict = toml.as_data + read_toml_dict = toml.as_object # TODO: replace by this error when using tomlkit only in the future: # except TOMLKitError as err: except TomlDecodeError as err: @@ -194,7 +194,7 @@ def merge_toml_dict(self) -> JsonDict: merged_dict = self._blender.mix() # TODO: check if the merged style file is still needed merged_style_path: Path = self.cache_dir / MERGED_STYLE_TOML - toml = TOMLFormat(data=merged_dict) + toml = TomlDoc(obj=merged_dict) attempt = 1 while attempt < 5: @@ -215,7 +215,7 @@ def file_field_pair(filename: str, base_file_class: Type[NitpickPlugin]) -> Dict if base_file_class.validation_schema: file_field = fields.Nested(base_file_class.validation_schema, **kwargs) else: - # For some files (e.g.: pyproject.toml, INI files), there is no strict schema; + # For some files (e.g.: TOML/ INI files), there is no strict schema; # it can be anything they allow. # It's out of Nitpick's scope to validate those files. file_field = fields.Dict(fields.String, **kwargs) diff --git a/src/nitpick/style/fetchers/pypackage.py b/src/nitpick/style/fetchers/pypackage.py index 5d1887dc..27c19d60 100644 --- a/src/nitpick/style/fetchers/pypackage.py +++ b/src/nitpick/style/fetchers/pypackage.py @@ -4,15 +4,9 @@ from typing import Tuple from urllib.parse import urlparse +from nitpick import compat from nitpick.style.fetchers.base import StyleFetcher -try: - from importlib.abc import Traversable # type: ignore[attr-defined] - from importlib.resources import files # type: ignore[attr-defined] -except ImportError: - from importlib_resources import files - from importlib_resources.abc import Traversable - @dataclass(unsafe_hash=True) class PythonPackageURL: @@ -37,9 +31,9 @@ def parse_url(cls, url: str) -> "PythonPackageURL": return cls(import_path=import_path, resource_name=resource_name) @property - def raw_content_url(self) -> Traversable: + def raw_content_url(self) -> compat.Traversable: """Raw path of resource file.""" - return files(self.import_path).joinpath(self.resource_name) + return compat.files(self.import_path).joinpath(self.resource_name) @dataclass(repr=True, unsafe_hash=True) diff --git a/src/nitpick/typedefs.py b/src/nitpick/typedefs.py index 25a7523b..341e945b 100644 --- a/src/nitpick/typedefs.py +++ b/src/nitpick/typedefs.py @@ -1,4 +1,5 @@ """Type definitions.""" +from collections import OrderedDict from pathlib import Path from typing import Any, Dict, Iterable, List, Tuple, Type, Union @@ -9,7 +10,8 @@ StrOrList = Union[str, List[str]] StrOrIterable = Union[str, Iterable[str]] Flake8Error = Tuple[int, int, str, Type] -YamlData = Union[CommentedSeq, CommentedMap] +YamlObject = Union[CommentedSeq, CommentedMap] +YamlValue = Union[JsonDict, OrderedDict, List[Any], str, float] # Decorated property not supported · Issue #1362 · python/mypy # https://github.com/python/mypy/issues/1362#issuecomment-562141376 diff --git a/styles/pre-commit/general.toml b/styles/pre-commit/general.toml deleted file mode 100644 index 646ddeab..00000000 --- a/styles/pre-commit/general.toml +++ /dev/null @@ -1,7 +0,0 @@ -[[".pre-commit-config.yaml".repos]] -yaml = """ - - repo: https://github.com/pre-commit/pre-commit-hooks - hooks: - - id: end-of-file-fixer - - id: trailing-whitespace -""" diff --git a/tests/helpers.py b/tests/helpers.py index 2b35999c..56d6cc38 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -23,13 +23,13 @@ SETUP_CFG, ) from nitpick.core import Nitpick +from nitpick.documents import TomlDoc from nitpick.flake8 import NitpickFlake8Extension -from nitpick.formats import TOMLFormat from nitpick.plugins.pre_commit import PreCommitPlugin from nitpick.typedefs import Flake8Error, PathOrStr, StrOrList from nitpick.violations import Fuss, Reporter -STYLES_DIR: Path = Path(__file__).parent.parent / "styles" +STYLES_DIR: Path = Path(__file__).parent.parent / "src" / "nitpick" / "resources" # Non-breaking space NBSP = "\xc2\xa0" @@ -48,6 +48,17 @@ def assert_conditions(*args): raise AssertionError() +def from_path_or_str(file_contents: PathOrStr): + """Read file contents from a Path or string.""" + if file_contents is None: + raise RuntimeError("No path and no file contents.") + + if isinstance(file_contents, Path): + return file_contents.read_text() + + return file_contents + + class ProjectMock: """A mocked Python project to help on tests.""" @@ -83,7 +94,7 @@ def create_symlink(self, link_name: str, target_dir: Path, target_file: str = No self.files_to_lint.append(path) return self - def _simulate_run(self, *partial_names: str, offline=False, api=True, flake8=True, fix=False) -> "ProjectMock": + def _simulate_run(self, *partial_names: str, offline=False, api=True, flake8=True, autofix=False) -> "ProjectMock": """Simulate a manual flake8 run and using the API. - Clear the singleton cache. @@ -96,7 +107,7 @@ def _simulate_run(self, *partial_names: str, offline=False, api=True, flake8=Tru self.nitpick_instance = Nitpick.singleton().init(offline=offline) if api: - self._actual_violations = set(self.nitpick_instance.run(*partial_names, fix=fix)) + self._actual_violations = set(self.nitpick_instance.run(*partial_names, autofix=autofix)) if flake8: npc = NitpickFlake8Extension(filename=str(self.files_to_lint[0])) @@ -117,22 +128,22 @@ def flake8(self, offline=False): def api_check(self, *partial_names: str, offline=False): """Test only the API in check mode, no flake8 plugin.""" - return self._simulate_run(*partial_names, offline=offline, api=True, flake8=False, fix=False) + return self._simulate_run(*partial_names, offline=offline, api=True, flake8=False, autofix=False) def api_fix(self, *partial_names: str): - """Test only the API in fix mode, no flake8 plugin.""" - return self._simulate_run(*partial_names, api=True, flake8=False, fix=True) + """Test only the API in autofix mode, no flake8 plugin.""" + return self._simulate_run(*partial_names, api=True, flake8=False, autofix=True) def api_check_then_fix( self, *expected_violations_when_fixing: Fuss, partial_names: Optional[Iterable[str]] = None ) -> "ProjectMock": - """Assert that check mode does not change files, and that fix mode changes them. + """Assert that check mode does not change files, and that autofix mode changes them. Perform a series of calls and assertions: 1. Call the API in check mode, assert violations, assert files contents were not modified. - 2. Call the API in fix mode and assert violations again. + 2. Call the API in autofix mode and assert violations again. - :param expected_violations_when_fixing: Expected violations when "fix mode" is on. + :param expected_violations_when_fixing: Expected violations when "autofix mode" is on. :param partial_names: Names of the files to enforce configs for. :return: ``self`` for method chaining (fluent interface) """ @@ -171,7 +182,7 @@ def read_multiple_files(self, filenames: Iterable[PathOrStr]) -> Dict[PathOrStr, """Read multiple files and return a hash with filename (key) and contents (value).""" return {filename: self.read_file(filename) for filename in filenames} - def save_file(self, filename: PathOrStr, file_contents: str, lint: bool = None) -> "ProjectMock": + def save_file(self, filename: PathOrStr, file_contents: PathOrStr, lint: bool = None) -> "ProjectMock": """Save a file in the root dir with the desired contents and a new line at the end. Create the parent dirs if the file name contains a slash. @@ -188,7 +199,8 @@ def save_file(self, filename: PathOrStr, file_contents: str, lint: bool = None) path.parent.mkdir(parents=True, exist_ok=True) if lint or path.suffix == ".py": self.files_to_lint.append(path) - clean = dedent(file_contents).strip() + clean = dedent(from_path_or_str(file_contents)).strip() + path.parent.mkdir(parents=True, exist_ok=True) path.write_text(f"{clean}\n") return self @@ -196,9 +208,9 @@ def touch_file(self, filename: PathOrStr) -> "ProjectMock": """Save an empty file (like the ``touch`` command).""" return self.save_file(filename, "") - def style(self, file_contents: str) -> "ProjectMock": + def style(self, file_contents: PathOrStr) -> "ProjectMock": """Save the default style file.""" - return self.save_file(NITPICK_STYLE_TOML, file_contents) + return self.save_file(NITPICK_STYLE_TOML, from_path_or_str(file_contents)) # TODO: remove this function, don't test real styles anymore to avoid breaking tests on Renovate updates def load_styles(self, *args: PathOrStr) -> "ProjectMock": @@ -212,26 +224,26 @@ def load_styles(self, *args: PathOrStr) -> "ProjectMock": self.named_style(filename, style_path.read_text()) return self - def named_style(self, filename: PathOrStr, file_contents: str) -> "ProjectMock": + def named_style(self, filename: PathOrStr, file_contents: PathOrStr) -> "ProjectMock": """Save a style file with a name. Add the .toml extension if needed.""" - return self.save_file(self.ensure_toml_extension(filename), file_contents) + return self.save_file(self.ensure_toml_extension(filename), from_path_or_str(file_contents)) @staticmethod def ensure_toml_extension(filename: PathOrStr) -> PathOrStr: """Ensure a file name has the .toml extension.""" return filename if str(filename).endswith(".toml") else f"{filename}.toml" - def setup_cfg(self, file_contents: str) -> "ProjectMock": + def setup_cfg(self, file_contents: PathOrStr) -> "ProjectMock": """Save setup.cfg.""" - return self.save_file(SETUP_CFG, file_contents) + return self.save_file(SETUP_CFG, from_path_or_str(file_contents)) - def pyproject_toml(self, file_contents: str) -> "ProjectMock": + def pyproject_toml(self, file_contents: PathOrStr) -> "ProjectMock": """Save pyproject.toml.""" - return self.save_file(PYPROJECT_TOML, file_contents) + return self.save_file(PYPROJECT_TOML, from_path_or_str(file_contents)) - def pre_commit(self, file_contents: str) -> "ProjectMock": + def pre_commit(self, file_contents: PathOrStr) -> "ProjectMock": """Save .pre-commit-config.yaml.""" - return self.save_file(PreCommitPlugin.filename, file_contents) + return self.save_file(PreCommitPlugin.filename, from_path_or_str(file_contents)) def raise_assertion_error(self, expected_error: str, assertion_message: str = None): """Show detailed errors in case of an assertion failure.""" @@ -280,9 +292,9 @@ def assert_no_errors(self) -> "ProjectMock": def assert_merged_style(self, toml_string: str) -> "ProjectMock": """Assert the contents of the merged style file.""" - expected = TOMLFormat(path=self.cache_dir / MERGED_STYLE_TOML) - actual = TOMLFormat(string=dedent(toml_string)) - compare(expected.as_data, actual.as_data) + expected = TomlDoc(path=self.cache_dir / MERGED_STYLE_TOML) + actual = TomlDoc(string=dedent(toml_string)) + compare(expected.as_object, actual.as_object) return self def assert_violations(self, *expected_violations: Fuss, disclaimer="") -> "ProjectMock": @@ -341,7 +353,7 @@ def _simulate_cli(self, command: str, expected_str_or_lines: StrOrList = None, * def cli_run( self, expected_str_or_lines: StrOrList = None, - fix=False, + autofix=False, violations=0, exception_class=None, exit_code: int = None, @@ -350,7 +362,7 @@ def cli_run( if exit_code is None: exit_code = 1 if expected_str_or_lines else 0 result, actual, expected = self._simulate_cli( - "fix" if fix else "check", expected_str_or_lines, exit_code=exit_code + "fix" if autofix else "check", expected_str_or_lines, exit_code=exit_code ) if exception_class: assert isinstance(result.exception, exception_class) @@ -392,7 +404,7 @@ def assert_file_contents(self, *name_contents: Union[PathOrStr, Optional[str]]) assert len(name_contents) % 2 == 0, "Supply pairs of arguments: filename (PathOrStr) and file contents (str)" for filename, file_contents in windowed(name_contents, 2, step=2): actual = self.read_file(filename) - expected = None if file_contents is None else dedent(file_contents).lstrip() + expected = None if file_contents is None else dedent(from_path_or_str(file_contents)).lstrip() compare(actual=actual, expected=expected, prefix=f"Filename: {filename}") return self diff --git a/tests/test_ini.py b/tests/test_ini.py index 08c6a05a..2d676d7c 100644 --- a/tests/test_ini.py +++ b/tests/test_ini.py @@ -35,12 +35,12 @@ def test_default_style_is_applied(project_default): skip = .tox,build [mypy] - follow_imports = skip + follow_imports = normal ignore_missing_imports = True strict_optional = True warn_no_return = True warn_redundant_casts = True - warn_unused_ignores = True + warn_unused_ignores = False """ expected_editor_config = """ root = True diff --git a/tests/test_json.py b/tests/test_json.py index 256996e5..ebd5ca84 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -4,7 +4,7 @@ import pytest from nitpick.constants import PACKAGE_JSON, READ_THE_DOCS_URL -from nitpick.plugins.json import JSONPlugin +from nitpick.plugins.json import JsonPlugin from nitpick.violations import Fuss, SharedViolations from tests.helpers import ProjectMock @@ -93,7 +93,7 @@ def test_missing_different_values_with_contains_json_with_contains_keys(tmp_path Fuss( True, PACKAGE_JSON, - SharedViolations.MISSING_VALUES.code + JSONPlugin.violation_base_code, + SharedViolations.MISSING_VALUES.code + JsonPlugin.violation_base_code, " has missing values:", """ { @@ -110,7 +110,7 @@ def test_missing_different_values_with_contains_json_with_contains_keys(tmp_path Fuss( True, PACKAGE_JSON, - SharedViolations.DIFFERENT_VALUES.code + JSONPlugin.violation_base_code, + SharedViolations.DIFFERENT_VALUES.code + JsonPlugin.violation_base_code, " has different values. Use this:", """ { @@ -142,7 +142,7 @@ def test_missing_different_values_with_contains_json_without_contains_keys(tmp_p Fuss( True, "my.json", - SharedViolations.MISSING_VALUES.code + JSONPlugin.violation_base_code, + SharedViolations.MISSING_VALUES.code + JsonPlugin.violation_base_code, " has missing values:", """ { @@ -173,7 +173,7 @@ def test_missing_different_values_with_contains_json_without_contains_keys(tmp_p Fuss( True, "my.json", - SharedViolations.DIFFERENT_VALUES.code + JSONPlugin.violation_base_code, + SharedViolations.DIFFERENT_VALUES.code + JsonPlugin.violation_base_code, " has different values. Use this:", """ { diff --git a/tests/test_resources.py b/tests/test_resources.py new file mode 100644 index 00000000..27a5e5da --- /dev/null +++ b/tests/test_resources.py @@ -0,0 +1,18 @@ +"""Resource tests.""" +from pathlib import Path + +from identify.identify import ALL_TAGS + +from tests.helpers import STYLES_DIR + + +def test_packages_named_after_identify_tags(): + """Test if the directories are packages and also "identify" tags.""" + for item in Path(STYLES_DIR).glob("**/*"): + if not item.is_dir() or item.name in {"__pycache__", "any"}: + continue + + assert item.name in ALL_TAGS, f"The directory {item.name!r} is not a valid 'identify' tag" + init_py = list(item.glob("__init__.py")) + assert init_py, f"The directory {item.name!r} is not a Python package" + assert init_py[0].is_file() diff --git a/tests/test_toml.py b/tests/test_toml.py index 8f3579d2..432d7594 100644 --- a/tests/test_toml.py +++ b/tests/test_toml.py @@ -1,4 +1,4 @@ -"""pyproject.toml tests.""" +"""TOML tests.""" from nitpick.constants import PYPROJECT_TOML from nitpick.plugins.toml import TomlPlugin from nitpick.violations import Fuss, SharedViolations diff --git a/tests/test_yaml.py b/tests/test_yaml.py new file mode 100644 index 00000000..0b45d612 --- /dev/null +++ b/tests/test_yaml.py @@ -0,0 +1,75 @@ +"""YAML tests.""" +from nitpick.plugins.yaml import YamlPlugin +from nitpick.violations import Fuss, SharedViolations +from tests.helpers import ProjectMock + + +def test_suggest_initial_contents(tmp_path, datadir): + """Suggest contents when YAML files do not exist.""" + filename = ".github/workflows/python.yaml" + expected_yaml = (datadir / "new-expected.yaml").read_text() + ProjectMock(tmp_path).style(datadir / "new-desired.toml").api_check_then_fix( + Fuss( + True, + filename, + SharedViolations.CREATE_FILE_WITH_SUGGESTION.code + YamlPlugin.violation_base_code, + " was not found. Create it with this content:", + expected_yaml, + ) + ).assert_file_contents(filename, expected_yaml) + + +def test_missing_different_values(tmp_path, datadir): + """Test different and missing values on any YAML.""" + filename = "me/deep/rooted.yaml" + ProjectMock(tmp_path).save_file(filename, datadir / "existing-actual.yaml").style( + datadir / "existing-desired.toml" + ).api_check_then_fix( + Fuss( + True, + filename, + YamlPlugin.violation_base_code + SharedViolations.DIFFERENT_VALUES.code, + " has different values. Use this:", + """ + mixed: + - lets: + ruin: this + with: + - weird + - '1' + - crap + - second item: also a dict + - c: 1 + b: 2 + a: 3 + python: + install: + - extra_requirements: + - some + - nice + - package + version: '3.9' + """, + ), + Fuss( + True, + filename, + YamlPlugin.violation_base_code + SharedViolations.MISSING_VALUES.code, + " has missing values:", + """ + root_key: + a_dict: + - c: '3.1' + - b: 2 + - a: string value + a_nested: + int: 10 + list: + - 0 + - 2 + - 1 + """, + ), + ).assert_file_contents( + filename, datadir / "existing-expected.yaml" + ) diff --git a/tests/test_yaml/existing-actual.yaml b/tests/test_yaml/existing-actual.yaml new file mode 100644 index 00000000..516e975d --- /dev/null +++ b/tests/test_yaml/existing-actual.yaml @@ -0,0 +1,16 @@ +# Root comment +python: + version: 3.6 # Python 3.6 EOL this month! + install: + - method: pip + path: . + extra_requirements: + - doc +mixed: + - 1 + - string + - c: 1 + b: 2 + a: 3 + - and the remaining items are untouched + - [ 5,3,1 ] diff --git a/tests/test_yaml/existing-desired.toml b/tests/test_yaml/existing-desired.toml new file mode 100644 index 00000000..f24dc87f --- /dev/null +++ b/tests/test_yaml/existing-desired.toml @@ -0,0 +1,34 @@ +["me/deep/rooted.yaml".python] +version = "3.9" + +[["me/deep/rooted.yaml".python.install]] +extra_requirements = ["some", "nice", "package"] + +[["me/deep/rooted.yaml".root_key.a_dict]] +c = "3.1" + +[["me/deep/rooted.yaml".root_key.a_dict]] +b = 2 + +[["me/deep/rooted.yaml".mixed]] +lets = { ruin = "this", with = ["weird", "1", "crap"] } + +[["me/deep/rooted.yaml".mixed]] +"second item" = "also a dict" + +# Even though the third item is the same, +# it will show up as "has different values". +# When it's a list, the diff comparison returns the whole list +# and doesn't compare individual items. +# TODO: Maybe an improvement for the future? Compare and discard list items that are equal +[["me/deep/rooted.yaml".mixed]] +c = 1 +b = 2 +a = 3 + +[["me/deep/rooted.yaml".root_key.a_dict]] +a = "string value" + +["me/deep/rooted.yaml".root_key.a_nested] +list = [0, 2, 1] +int = 10 diff --git a/tests/test_yaml/existing-expected.yaml b/tests/test_yaml/existing-expected.yaml new file mode 100644 index 00000000..ee899158 --- /dev/null +++ b/tests/test_yaml/existing-expected.yaml @@ -0,0 +1,34 @@ +# Root comment +python: + version: '3.9' # Python 3.6 EOL this month! + install: + - method: pip + path: . + extra_requirements: + - some + - nice + - package +mixed: + - lets: + ruin: this + with: + - weird + - '1' + - crap + - second item: also a dict + - c: 1 + b: 2 + a: 3 + - and the remaining items are untouched + - [5, 3, 1] +root_key: + a_dict: + - c: '3.1' + - b: 2 + - a: string value + a_nested: + int: 10 + list: + - 0 + - 2 + - 1 diff --git a/tests/test_yaml/new-desired.toml b/tests/test_yaml/new-desired.toml new file mode 100644 index 00000000..2d9c5426 --- /dev/null +++ b/tests/test_yaml/new-desired.toml @@ -0,0 +1,18 @@ +# TOML tables to represent YAML lists +[[".github/workflows/python.yaml".jobs.build.steps]] +uses = "actions/checkout@v2" + +[[".github/workflows/python.yaml".jobs.build.steps]] +name = "Set up Python ${{ matrix.python-version }}" +uses = "actions/setup-python@v2" + +# A dynamic inline table; this causes issues with the TOML decoder +# See https://github.com/uiri/toml/issues/362 +with = {"python-version" = "${{ matrix.python-version }}"} + +[".github/workflows/python.yaml".jobs.build.strategy.matrix] +os = ["ubuntu-latest", "macos-latest", "windows-latest"] +"python-version" = ["3.6", "3.7", "3.8", "3.9", "3.10"] + +[".github/workflows/python.yaml".jobs.build] +"runs-on" = "${{ matrix.os }}" diff --git a/tests/test_yaml/new-expected.yaml b/tests/test_yaml/new-expected.yaml new file mode 100644 index 00000000..b3ec2850 --- /dev/null +++ b/tests/test_yaml/new-expected.yaml @@ -0,0 +1,21 @@ +jobs: + build: + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + strategy: + matrix: + os: + - ubuntu-latest + - macos-latest + - windows-latest + python-version: + - '3.6' + - '3.7' + - '3.8' + - '3.9' + - '3.10'