Skip to content

Commit

Permalink
Change linter to ruff
Browse files Browse the repository at this point in the history
  • Loading branch information
vemel committed Apr 24, 2024
1 parent b8c05b0 commit 3dfb015
Show file tree
Hide file tree
Showing 14 changed files with 101 additions and 45 deletions.
10 changes: 2 additions & 8 deletions .github/workflows/on_push.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ jobs:
include:
- version: "3.10"
check-lint: "true"
check-formatting: "true"
check-typing: "true"
- version: "3.11"
check-lint: "true"
Expand All @@ -41,13 +40,8 @@ jobs:
- name: Lint
if: ${{ matrix.check-lint }}
run: |
poetry run flake8 mypy_boto3_builder
- name: Formatting
if: ${{ matrix.check-formatting }}
run: |
poetry run black --check mypy_boto3_builder tests
poetry run isort -c mypy_boto3_builder tests
- run: echo "$(poetry env info --path)/bin" >> $GITHUB_PATH
poetry run ruff check mypy_boto3_builder
poetry run ruff check scripts
- name: Set up Node
if: ${{ matrix.check-typing }}
uses: actions/setup-node@v4
Expand Down
6 changes: 2 additions & 4 deletions .vscode/extensions.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,7 @@
"recommendations": [
"samuelcolvin.jinjahtml",
"tamasfe.even-better-toml",
"ms-python.black-formatter",
"ms-python.isort",
"ms-python.vscode-pylance",
"ms-python.flake8"
"charliermarsh.ruff",
"ms-python.vscode-pylance"
]
}
5 changes: 1 addition & 4 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,7 @@
{
"[python]": {
"editor.formatOnSave": true,
"editor.codeActionsOnSave": {
"source.organizeImports": "explicit"
},
"editor.defaultFormatter": "ms-python.black-formatter"
"editor.defaultFormatter": "charliermarsh.ruff"
},
"python.testing.pytestArgs": [],
"python.testing.unittestEnabled": false,
Expand Down
4 changes: 2 additions & 2 deletions mypy_boto3_builder/parsers/wrapper_package_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ class WrapperPackageParser:
package -- Prepared wrapper package with data
"""

init_arguments = [
init_arguments = (
Argument("region_name", TypeSubscript(Type.Optional, [Type.str]), Type.Ellipsis),
Argument("api_version", TypeSubscript(Type.Optional, [Type.str]), Type.Ellipsis),
Argument("use_ssl", TypeSubscript(Type.Optional, [Type.bool]), Type.Ellipsis),
Expand All @@ -46,7 +46,7 @@ class WrapperPackageParser:
TypeSubscript(Type.Optional, [ExternalImport.from_class(Config)]),
Type.Ellipsis,
),
]
)

def __init__(self, session: Session, package: WrapperPackage) -> None:
self.session = session
Expand Down
3 changes: 2 additions & 1 deletion mypy_boto3_builder/postprocessors/aiobotocore.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"""

from collections.abc import Iterator
from typing import Mapping

from boto3.dynamodb.table import BatchWriter, TableResource
from boto3.resources.base import ServiceResource
Expand Down Expand Up @@ -43,7 +44,7 @@ class AioBotocorePostprocessor(BasePostprocessor):
"page_size",
)

EXTERNAL_IMPORTS_MAP = {
EXTERNAL_IMPORTS_MAP: Mapping[ExternalImport, ExternalImport] = {
ExternalImport.from_class(StreamingBody): ExternalImport(
ImportString("aiobotocore", "response"), "StreamingBody"
),
Expand Down
4 changes: 2 additions & 2 deletions mypy_boto3_builder/service_name.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
Description for boto3 service.
"""

from typing import Literal
from typing import ClassVar, Literal

from mypy_boto3_builder.utils.strings import get_anchor_link, is_reserved

Expand Down Expand Up @@ -171,7 +171,7 @@ class ServiceNameCatalog:
)
old_ssm_sap = ServiceName("ssm-sap", "SsmSap", "ssmsap")

ITEMS: dict[str, ServiceName] = {
ITEMS: ClassVar[dict[str, ServiceName]] = {
ec2.boto3_name: ec2,
iam.boto3_name: iam,
s3.boto3_name: s3,
Expand Down
6 changes: 4 additions & 2 deletions mypy_boto3_builder/type_annotations/type_annotation.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
Wrapper for `typing` type annotation.
"""

from typing import Mapping

from typing_extensions import Self

from mypy_boto3_builder.import_helpers.import_record import ImportRecord
Expand All @@ -21,7 +23,7 @@ class TypeAnnotation(FakeAnnotation):
_typing_extensions = ImportString("typing_extensions")

# Set of supported type annotations. value is default import module
SUPPORTED_TYPES: dict[str, ImportString] = {
SUPPORTED_TYPES: Mapping[str, ImportString] = {
"Union": _typing, # typing.Union
"Any": _typing, # typing.Any
"Dict": _typing, # typing.Dict
Expand All @@ -44,7 +46,7 @@ class TypeAnnotation(FakeAnnotation):
}

# Set of fallback type annotations
FALLBACK: dict[str, tuple[tuple[int, int] | None, ImportString]] = {
FALLBACK: Mapping[str, tuple[tuple[int, int] | None, ImportString]] = {
"NotRequired": ((3, 12), _typing_extensions),
"TypedDict": ((3, 12), _typing_extensions),
"Literal": ((3, 12), _typing_extensions),
Expand Down
33 changes: 20 additions & 13 deletions mypy_boto3_builder/utils/strings.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,12 +66,10 @@ def get_short_docstring(doc: str) -> str:
return ""
for line in doc.splitlines():
line = line.strip().removesuffix("::")
if line.startswith(":"):
break
if line.lower().startswith("**request syntax**"):
break
if not line:
continue
if line.startswith(":") or line.lower().startswith("**request syntax**"):
break
if ". " in line:
result.append(line.split(". ")[0])
break
Expand All @@ -80,19 +78,28 @@ def get_short_docstring(doc: str) -> str:
break

result_str = " ".join(result).replace("```", "`").replace("``", "`").strip()
if result_str.count("`") % 2:
result_str = f"{result_str}`"
if result_str and not result_str.endswith("."):
result_str = f"{result_str}."
result_str = clean_artifacts(result_str)

return textwrap(result_str, width=80)

if "<https:" in result_str:
result_str = AWS_LINK_RE.sub(r"[\g<1>](https://\g<2>)", result_str)

def clean_artifacts(line: str) -> str:
"""
Remove common artifacts in botocre docs.
"""
if line.count("`") % 2:
line = f"{line}`"
if line and not line.endswith("."):
line = f"{line}."

if "<https:" in line:
line = AWS_LINK_RE.sub(r"[\g<1>](https://\g<2>)", line)
# FIXME: temporary fix for pca-connector-ad service
result_str = result_str.replace("https\\:", "https:")
line = line.replace("https\\:", "https:")
# FIXME: temporary fix for neptunedata service
result_str = result_str.replace("neptune-db\\:", "neptune-db:")
line = line.replace("neptune-db\\:", "neptune-db:")

return textwrap(result_str, width=80)
return line


def textwrap(text: str, width: int) -> str:
Expand Down
47 changes: 47 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -157,3 +157,50 @@ ignore_errors = true
max-line-length = 100
exclude = [".git", "__pycache__", "docs", "build", "dist", "test_*.py", "*.pyi"]
ignore = ["E704", "W503"]

[tool.ruff]
exclude = [
".eggs",
".git",
".git-rewrite",
".mypy_cache",
".pytest_cache",
".ruff_cache",
".venv",
".vscode",
"build",
"dist",
]

# Same as Black.
line-length = 100
indent-width = 4
target-version = "py310"

[tool.ruff.lint]
select = ["E4", "E7", "E9", "F", "B", "I", "N", "D", "C4", "C90", "RUF"]
ignore = [
"E501",
"N803",
"N818",
"D107",
"D200",
"D203",
"D212",
"D406",
"D407",
"D413",
"D417",
]
fixable = ["ALL"]
unfixable = ["B"]
dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
exclude = ["test_*.py", "*.pyi"]

[tool.ruff.format]
quote-style = "double"
indent-style = "space"
skip-magic-trailing-comma = false
line-ending = "auto"
docstring-code-format = false
docstring-code-line-length = "dynamic"
6 changes: 2 additions & 4 deletions scripts/before_commit.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,9 @@ cd $ROOT_PATH
# poetry run vulture mypy_boto3_builder --make-whitelist > vulture_whitelist.txt
poetry run vulture mypy_boto3_builder vulture_whitelist.txt
poetry run npx pyright mypy_boto3_builder
poetry run flake8 mypy_boto3_builder
poetry run pytest
poetry run black .
poetry run isort .
poetry run mypy mypy_boto3_builder
poetry run ruff check mypy_boto3_builder
poetry run ruff check scripts
# poetry run pytest --cov-report html --cov mypy_boto3_builder

# ./scripts/docs.sh
3 changes: 2 additions & 1 deletion scripts/check_output.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
- [x] pyright
- [x] mypy
"""

import argparse
import json
import logging
Expand Down Expand Up @@ -158,7 +159,7 @@ def run_flake8(path: Path) -> None:
except subprocess.CalledProcessError:
temp_path = Path(f.name)
output = temp_path.read_text()
raise SnapshotMismatchError(output)
raise SnapshotMismatchError(output) from None


def run_pyright(path: Path) -> None:
Expand Down
2 changes: 1 addition & 1 deletion scripts/open.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,4 @@ set -e

SITE_PACKAGES=`poetry run python -c "import sys; import os; paths = filter(lambda x: x.startswith(os.getcwd()), sys.path); print(list(paths)[0])"`
echo "Site packages:" $SITE_PACKAGES
code $SITE_PACKAGES/boto3
code $SITE_PACKAGES/botocore
6 changes: 3 additions & 3 deletions scripts/release.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
"""
Publish packages to PyPI.
"""

import argparse
import logging
import os
Expand Down Expand Up @@ -161,9 +162,8 @@ def build(path: Path, max_retries: int = 10) -> Path:
with chdir(path):
check_call([sys.executable, "setup.py", "build", "sdist", "bdist_wheel"])

whl_path = list((path / "dist").glob("*.whl"))[0]

tar_path = list((path / "dist").glob("*.tar.gz"))[0]
whl_path = next(iter((path / "dist").glob("*.whl")))
tar_path = next(iter((path / "dist").glob("*.tar.gz")))
check_call(["tar", "-tzf", tar_path.as_posix()])
check_call([sys.executable, "-m", "zipfile", "--list", whl_path.as_posix()])
except (subprocess.CalledProcessError, IndexError) as e:
Expand Down
11 changes: 11 additions & 0 deletions tests/parsers/test_shape_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,10 @@ def test_get_paginator_names(self) -> None:
def test_get_client_method_map(self, ServiceModelMock: MagicMock) -> None:
session_mock = MagicMock()
service_name_mock = MagicMock()
operation_model_mock = MagicMock()
operation_model_mock.output_shape.serialization = {}
ServiceModelMock().operation_names = ["my_operation"]
ServiceModelMock().operation_model.return_value = operation_model_mock
session_mock._loader.load_service_model.return_value = {
"resources": {"c": None, "a": None, "b": None}
}
Expand All @@ -51,6 +54,10 @@ def test_get_paginate_method(self, ServiceModelMock: MagicMock) -> None:
operation_model_mock = MagicMock()
required_arg_shape_mock = MagicMock()
optional_arg_shape_mock = MagicMock()
required_arg_shape_mock.serialization = {}
optional_arg_shape_mock.serialization = {}
operation_model_mock.output_shape.serialization = {}
operation_model_mock.input_shape.serialization = {}
operation_model_mock.input_shape.members.items.return_value = [
(
"required_arg",
Expand Down Expand Up @@ -92,6 +99,10 @@ def test_get_collection_filter_method(self, ServiceModelMock: MagicMock) -> None
operation_model_mock = MagicMock()
required_arg_shape_mock = MagicMock()
optional_arg_shape_mock = MagicMock()
required_arg_shape_mock.serialization = {}
optional_arg_shape_mock.serialization = {}
operation_model_mock.output_shape.serialization = {}
operation_model_mock.input_shape.serialization = {}
operation_model_mock.input_shape.required_members = ["required_arg"]
operation_model_mock.input_shape.members.items.return_value = [
(
Expand Down

0 comments on commit 3dfb015

Please sign in to comment.