From f7282c5a1342092e910c6e6b7e419d9c91c5bc40 Mon Sep 17 00:00:00 2001 From: konstin Date: Thu, 18 May 2023 16:36:14 +0200 Subject: [PATCH 01/12] Lint pyproject.toml This adds a new rule `InvalidPyprojectToml` that lints pyproject.toml by checking if https://github.com/PyO3/pyproject-toml-rs can parse it. This means the linting is currently very basic, e.g. we don't check whether the name is actually a valid python project name or appropriately normalized. It does catch errors e.g. with invalid dependency requirements or problems withs the license specifications. It is open to be extended in the future (validate name, SPDX expressions, classifiers, ...), either in ruff or in pyproject-toml-rs. TODOs: - [ ] Why does `FilePattern::Builtin("pyproject.toml")` alone not work? I've added `*.toml` for now to test but this should be changed before merging (help wanted) - [ ] Run this over the ecosystem CI dataset --- Cargo.lock | 33 +++++++++++ crates/ruff/Cargo.toml | 1 + .../ruff/pyproject_toml/bleach/pyproject.toml | 7 +++ .../invalid_author/pyproject.toml | 7 +++ .../pyproject_toml/maturin/pyproject.toml | 57 +++++++++++++++++++ .../maturin_gh_1615/pyproject.toml | 39 +++++++++++++ crates/ruff/src/codes.rs | 1 + crates/ruff/src/lib.rs | 1 + crates/ruff/src/pyproject_toml.rs | 41 +++++++++++++ crates/ruff/src/registry.rs | 1 + crates/ruff/src/rules/ruff/mod.rs | 19 ++++++- .../ruff/rules/invalid_pyproject_toml.rs | 45 +++++++++++++++ crates/ruff/src/rules/ruff/rules/mod.rs | 2 + ...ff__rules__ruff__tests__RUF200_bleach.snap | 17 ++++++ ...s__ruff__tests__RUF200_invalid_author.snap | 13 +++++ ...f__rules__ruff__tests__RUF200_maturin.snap | 4 ++ ...__ruff__tests__RUF200_maturin_gh_1615.snap | 14 +++++ crates/ruff/src/settings/defaults.rs | 10 +++- crates/ruff_cli/src/diagnostics.rs | 13 +++++ crates/ruff_python_stdlib/src/path.rs | 6 ++ 20 files changed, 328 insertions(+), 3 deletions(-) create mode 100644 crates/ruff/resources/test/fixtures/ruff/pyproject_toml/bleach/pyproject.toml create mode 100644 crates/ruff/resources/test/fixtures/ruff/pyproject_toml/invalid_author/pyproject.toml create mode 100644 crates/ruff/resources/test/fixtures/ruff/pyproject_toml/maturin/pyproject.toml create mode 100644 crates/ruff/resources/test/fixtures/ruff/pyproject_toml/maturin_gh_1615/pyproject.toml create mode 100644 crates/ruff/src/pyproject_toml.rs create mode 100644 crates/ruff/src/rules/ruff/rules/invalid_pyproject_toml.rs create mode 100644 crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_bleach.snap create mode 100644 crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_invalid_author.snap create mode 100644 crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_maturin.snap create mode 100644 crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_maturin_gh_1615.snap diff --git a/Cargo.lock b/Cargo.lock index 10429702abdda..b5e794e7af3e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -913,6 +913,7 @@ checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown", + "serde", ] [[package]] @@ -1401,6 +1402,22 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "pep508_rs" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "969679a29dfdc8278a449f75b3dd45edf57e649bd59f7502429c2840751c46d8" +dependencies = [ + "once_cell", + "pep440_rs", + "regex", + "serde", + "thiserror", + "tracing", + "unicode-width", + "url", +] + [[package]] name = "percent-encoding" version = "2.2.0" @@ -1563,6 +1580,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "pyproject-toml" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f04dbbb336bd88583943c7cd973a32fed323578243a7569f40cb0c7da673321b" +dependencies = [ + "indexmap", + "pep440_rs", + "pep508_rs", + "serde", + "toml", +] + [[package]] name = "quick-junit" version = "0.3.2" @@ -1751,6 +1781,7 @@ dependencies = [ "pathdiff", "pep440_rs", "pretty_assertions", + "pyproject-toml", "quick-junit", "regex", "result-like", @@ -2600,6 +2631,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if", + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -2771,6 +2803,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] diff --git a/crates/ruff/Cargo.toml b/crates/ruff/Cargo.toml index 1031796ac8ec5..3d91ccc6da28e 100644 --- a/crates/ruff/Cargo.toml +++ b/crates/ruff/Cargo.toml @@ -50,6 +50,7 @@ path-absolutize = { workspace = true, features = [ ] } pathdiff = { version = "0.2.1" } pep440_rs = { version = "0.3.1", features = ["serde"] } +pyproject-toml = { version = "0.6.0" } quick-junit = { version = "0.3.2" } regex = { workspace = true } result-like = { version = "0.4.6" } diff --git a/crates/ruff/resources/test/fixtures/ruff/pyproject_toml/bleach/pyproject.toml b/crates/ruff/resources/test/fixtures/ruff/pyproject_toml/bleach/pyproject.toml new file mode 100644 index 0000000000000..6345a9db16eed --- /dev/null +++ b/crates/ruff/resources/test/fixtures/ruff/pyproject_toml/bleach/pyproject.toml @@ -0,0 +1,7 @@ +[project] +name = "hello-world" +version = "0.1.0" +# There's a comma missing here +dependencies = [ + "tinycss2>=1.1.0<1.2", +] diff --git a/crates/ruff/resources/test/fixtures/ruff/pyproject_toml/invalid_author/pyproject.toml b/crates/ruff/resources/test/fixtures/ruff/pyproject_toml/invalid_author/pyproject.toml new file mode 100644 index 0000000000000..675ad7afe990e --- /dev/null +++ b/crates/ruff/resources/test/fixtures/ruff/pyproject_toml/invalid_author/pyproject.toml @@ -0,0 +1,7 @@ +[project] +name = "hello-world" +version = "0.1.0" +# Ensure that the spans from toml handle utf-8 correctly +authors = [ + { name = "콘스턴틴", email = 1 } +] diff --git a/crates/ruff/resources/test/fixtures/ruff/pyproject_toml/maturin/pyproject.toml b/crates/ruff/resources/test/fixtures/ruff/pyproject_toml/maturin/pyproject.toml new file mode 100644 index 0000000000000..6cbcadc76841e --- /dev/null +++ b/crates/ruff/resources/test/fixtures/ruff/pyproject_toml/maturin/pyproject.toml @@ -0,0 +1,57 @@ +# This is a valid pyproject.toml +# https://github.com/PyO3/maturin/blob/87ac3d9f74dd79ef2df9a20880b9f1fa23f9a437/pyproject.toml +[build-system] +requires = ["setuptools", "wheel>=0.36.2", "tomli>=1.1.0 ; python_version<'3.11'", "setuptools-rust>=1.4.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "maturin" +requires-python = ">=3.7" +classifiers = [ + "Topic :: Software Development :: Build Tools", + "Programming Language :: Rust", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] +dependencies = ["tomli>=1.1.0 ; python_version<'3.11'"] +dynamic = [ + "authors", + "description", + "license", + "readme", + "version" +] + +[project.optional-dependencies] +zig = [ + "ziglang~=0.10.0", +] +patchelf = [ + "patchelf", +] + +[project.urls] +"Source Code" = "https://github.com/PyO3/maturin" +Issues = "https://github.com/PyO3/maturin/issues" +Documentation = "https://maturin.rs" +Changelog = "https://maturin.rs/changelog.html" + +[tool.maturin] +bindings = "bin" + +[tool.black] +target_version = ['py37'] +extend-exclude = ''' +# Ignore cargo-generate templates +^/src/templates +''' + +[tool.ruff] +line-length = 120 +target-version = "py37" + +[tool.mypy] +disallow_untyped_defs = true +disallow_incomplete_defs = true +warn_no_return = true +ignore_missing_imports = true diff --git a/crates/ruff/resources/test/fixtures/ruff/pyproject_toml/maturin_gh_1615/pyproject.toml b/crates/ruff/resources/test/fixtures/ruff/pyproject_toml/maturin_gh_1615/pyproject.toml new file mode 100644 index 0000000000000..f566feabd690d --- /dev/null +++ b/crates/ruff/resources/test/fixtures/ruff/pyproject_toml/maturin_gh_1615/pyproject.toml @@ -0,0 +1,39 @@ +# license-files is wrong here +# https://github.com/PyO3/maturin/issues/1615 +[build-system] +requires = [ "maturin>=0.14", "numpy", "wheel", "patchelf",] +build-backend = "maturin" + +[project] +name = "..." +license-files = [ "license.txt",] +requires-python = ">=3.8" +requires-dist = [ "maturin>=0.14", "...",] +dependencies = [ "packaging", "...",] +zip-safe = false +version = "..." +readme = "..." +description = "..." +classifiers = [ "...",] +[[project.authors]] +name = "..." +email = "..." + +[project.urls] +homepage = "..." +documentation = "..." +repository = "..." + +[project.optional-dependencies] +test = [ "coverage", "...",] +docs = [ "sphinx", "sphinx-rtd-theme",] +devel = [] + +[tool.maturin] +include = [ "...",] +bindings = "pyo3" +compatability = "manylinux2014" + +[tool.pytest.ini_options] +testpaths = [ "...",] +addopts = "--color=yes --tb=native --cov-report term --cov-report html:docs/dist_coverage --cov=aisdb --doctest-modules --envfile .env" diff --git a/crates/ruff/src/codes.rs b/crates/ruff/src/codes.rs index 0fe8f7cb60b46..583f51e470b3b 100644 --- a/crates/ruff/src/codes.rs +++ b/crates/ruff/src/codes.rs @@ -722,6 +722,7 @@ pub fn code_to_rule(linter: Linter, code: &str) -> Option<(RuleGroup, Rule)> { (Ruff, "009") => (RuleGroup::Unspecified, Rule::FunctionCallInDataclassDefaultArgument), (Ruff, "010") => (RuleGroup::Unspecified, Rule::ExplicitFStringTypeConversion), (Ruff, "100") => (RuleGroup::Unspecified, Rule::UnusedNOQA), + (Ruff, "200") => (RuleGroup::Unspecified, Rule::InvalidPyprojectToml), // flake8-django (Flake8Django, "001") => (RuleGroup::Unspecified, Rule::DjangoNullableModelStringField), diff --git a/crates/ruff/src/lib.rs b/crates/ruff/src/lib.rs index 8c9e6772ce6ce..f1a9fe1be68da 100644 --- a/crates/ruff/src/lib.rs +++ b/crates/ruff/src/lib.rs @@ -27,6 +27,7 @@ pub mod logging; pub mod message; mod noqa; pub mod packaging; +pub mod pyproject_toml; pub mod registry; pub mod resolver; mod rule_redirects; diff --git a/crates/ruff/src/pyproject_toml.rs b/crates/ruff/src/pyproject_toml.rs new file mode 100644 index 0000000000000..19a1643747a30 --- /dev/null +++ b/crates/ruff/src/pyproject_toml.rs @@ -0,0 +1,41 @@ +use std::fs; +use std::path::Path; + +use anyhow::Result; +use pyproject_toml::PyProjectToml; +use ruff_text_size::{TextRange, TextSize}; + +use ruff_diagnostics::Diagnostic; +use ruff_python_ast::source_code::SourceFileBuilder; + +use crate::message::Message; +use crate::rules::ruff::rules::InvalidPyprojectToml; + +pub fn lint_pyproject_toml(path: &Path) -> Result> { + let contents = fs::read_to_string(path)?; + let source_file = SourceFileBuilder::new(path.to_string_lossy(), contents).finish(); + let err = match toml::from_str::(source_file.source_text()) { + Ok(_) => return Ok(Vec::default()), + Err(err) => err, + }; + + let range = match err.span() { + // This is bad but sometimes toml and/or serde just don't give us spans + None => TextRange::default(), + Some(range) => { + let expect_err = "pyproject.toml file be smaller than 4GB"; + TextRange::new( + TextSize::try_from(range.start).expect(expect_err), + TextSize::try_from(range.end).expect(expect_err), + ) + } + }; + + let toml_err = err.message().to_string(); + let diagnostic = Diagnostic::new(InvalidPyprojectToml { message: toml_err }, range); + Ok(vec![Message::from_diagnostic( + diagnostic, + source_file, + TextSize::default(), + )]) +} diff --git a/crates/ruff/src/registry.rs b/crates/ruff/src/registry.rs index 315d66ec66967..8a9cd9ede8127 100644 --- a/crates/ruff/src/registry.rs +++ b/crates/ruff/src/registry.rs @@ -646,6 +646,7 @@ ruff_macros::register_rules!( rules::ruff::rules::MutableDataclassDefault, rules::ruff::rules::FunctionCallInDataclassDefaultArgument, rules::ruff::rules::ExplicitFStringTypeConversion, + rules::ruff::rules::InvalidPyprojectToml, // flake8-django rules::flake8_django::rules::DjangoNullableModelStringField, rules::flake8_django::rules::DjangoLocalsInRenderFunction, diff --git a/crates/ruff/src/rules/ruff/mod.rs b/crates/ruff/src/rules/ruff/mod.rs index 3bc568f354f4a..8fae02be0f0c5 100644 --- a/crates/ruff/src/rules/ruff/mod.rs +++ b/crates/ruff/src/rules/ruff/mod.rs @@ -10,10 +10,11 @@ mod tests { use rustc_hash::FxHashSet; use test_case::test_case; + use crate::pyproject_toml::lint_pyproject_toml; use crate::registry::Rule; use crate::settings::resolve_per_file_ignores; use crate::settings::types::PerFileIgnore; - use crate::test::test_path; + use crate::test::{test_path, test_resource_path}; use crate::{assert_messages, settings}; #[test_case(Rule::ExplicitFStringTypeConversion, Path::new("RUF010.py"); "RUF010")] @@ -174,4 +175,20 @@ mod tests { assert_messages!(snapshot, diagnostics); Ok(()) } + + #[test_case(Rule::InvalidPyprojectToml, Path::new("bleach"))] + #[test_case(Rule::InvalidPyprojectToml, Path::new("invalid_author"))] + #[test_case(Rule::InvalidPyprojectToml, Path::new("maturin"))] + #[test_case(Rule::InvalidPyprojectToml, Path::new("maturin_gh_1615"))] + fn invalid_pyproject_toml(rule_code: Rule, path: &Path) -> Result<()> { + let snapshot = format!("{}_{}", rule_code.noqa_code(), path.to_string_lossy()); + let path = test_resource_path("fixtures") + .join("ruff") + .join("pyproject_toml") + .join(path) + .join("pyproject.toml"); + let messages = lint_pyproject_toml(&path)?; + assert_messages!(snapshot, messages); + Ok(()) + } } diff --git a/crates/ruff/src/rules/ruff/rules/invalid_pyproject_toml.rs b/crates/ruff/src/rules/ruff/rules/invalid_pyproject_toml.rs new file mode 100644 index 0000000000000..6410cb526dc85 --- /dev/null +++ b/crates/ruff/src/rules/ruff/rules/invalid_pyproject_toml.rs @@ -0,0 +1,45 @@ +use ruff_diagnostics::{AutofixKind, Violation}; +use ruff_macros::{derive_message_formats, violation}; + +/// ## What it does +/// Checks for any pyproject.toml that does not conform to the schema from the relevant PEPs. +/// +/// ## Why is this bad? +/// Your project may contain invalid metadata or configuration without you noticing +/// +/// ## Example +/// ```toml +/// [project] +/// name = "crab" +/// version = "1.0.0" +/// authors = ["Ferris the Crab "] +/// ``` +/// +/// Use instead: +/// ```toml +/// [project] +/// name = "crab" +/// version = "1.0.0" +/// authors = [ +/// { email = "ferris@example.org" }, +/// { name = "Ferris the Crab"} +/// ] +/// ``` +/// +/// ## References +/// - [Specification of `[project]` in pyproject.toml](https://packaging.python.org/en/latest/specifications/declaring-project-metadata/) +/// - [Specification of `[build-system]` in pyproject.toml](https://peps.python.org/pep-0518/) +/// - [Draft but implemented license declaration extensions](https://peps.python.org/pep-0639) +#[violation] +pub struct InvalidPyprojectToml { + pub message: String, +} + +impl Violation for InvalidPyprojectToml { + const AUTOFIX: AutofixKind = AutofixKind::None; + + #[derive_message_formats] + fn message(&self) -> String { + format!("Failed to parse pyproject.toml: {}", self.message) + } +} diff --git a/crates/ruff/src/rules/ruff/rules/mod.rs b/crates/ruff/src/rules/ruff/rules/mod.rs index 43543fe7cef40..62b75371fc63e 100644 --- a/crates/ruff/src/rules/ruff/rules/mod.rs +++ b/crates/ruff/src/rules/ruff/rules/mod.rs @@ -9,6 +9,7 @@ pub(crate) use collection_literal_concatenation::{ pub(crate) use explicit_f_string_type_conversion::{ explicit_f_string_type_conversion, ExplicitFStringTypeConversion, }; +pub(crate) use invalid_pyproject_toml::InvalidPyprojectToml; pub(crate) use mutable_defaults_in_dataclass_fields::{ function_call_in_dataclass_defaults, is_dataclass, mutable_dataclass_default, FunctionCallInDataclassDefaultArgument, MutableDataclassDefault, @@ -21,6 +22,7 @@ mod asyncio_dangling_task; mod collection_literal_concatenation; mod confusables; mod explicit_f_string_type_conversion; +mod invalid_pyproject_toml; mod mutable_defaults_in_dataclass_fields; mod pairwise_over_zipped; mod unused_noqa; diff --git a/crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_bleach.snap b/crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_bleach.snap new file mode 100644 index 0000000000000..f0fabf69b1b85 --- /dev/null +++ b/crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_bleach.snap @@ -0,0 +1,17 @@ +--- +source: crates/ruff/src/rules/ruff/mod.rs +--- +./resources/test/fixtures/ruff/pyproject_toml/bleach/pyproject.toml:5:16: RUF200 Failed to parse pyproject.toml: Version specifier `>=1.1.0<1.2` doesn't match PEP 440 rules +tinycss2>=1.1.0<1.2 + ^^^^^^^^^^^ + | +5 | version = "0.1.0" +6 | # There's a comma missing here +7 | dependencies = [ + | ________________^ +8 | | "tinycss2>=1.1.0<1.2", +9 | | ] + | |_^ RUF200 + | + + diff --git a/crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_invalid_author.snap b/crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_invalid_author.snap new file mode 100644 index 0000000000000..62d14287ac037 --- /dev/null +++ b/crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_invalid_author.snap @@ -0,0 +1,13 @@ +--- +source: crates/ruff/src/rules/ruff/mod.rs +--- +./resources/test/fixtures/ruff/pyproject_toml/invalid_author/pyproject.toml:6:30: RUF200 Failed to parse pyproject.toml: invalid type: integer `1`, expected a string + | +6 | # Ensure that the spans from toml handle utf-8 correctly +7 | authors = [ +8 | { name = "콘스턴틴", email = 1 } + | ^ RUF200 +9 | ] + | + + diff --git a/crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_maturin.snap b/crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_maturin.snap new file mode 100644 index 0000000000000..e51f71f81152a --- /dev/null +++ b/crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_maturin.snap @@ -0,0 +1,4 @@ +--- +source: crates/ruff/src/rules/ruff/mod.rs +--- + diff --git a/crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_maturin_gh_1615.snap b/crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_maturin_gh_1615.snap new file mode 100644 index 0000000000000..e136c2a10895d --- /dev/null +++ b/crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_maturin_gh_1615.snap @@ -0,0 +1,14 @@ +--- +source: crates/ruff/src/rules/ruff/mod.rs +--- +./resources/test/fixtures/ruff/pyproject_toml/maturin_gh_1615/pyproject.toml:9:17: RUF200 Failed to parse pyproject.toml: wanted string or table + | + 9 | [project] +10 | name = "..." +11 | license-files = [ "license.txt",] + | ^^^^^^^^^^^^^^^^^ RUF200 +12 | requires-python = ">=3.8" +13 | requires-dist = [ "maturin>=0.14", "...",] + | + + diff --git a/crates/ruff/src/settings/defaults.rs b/crates/ruff/src/settings/defaults.rs index 2aac4fa44f34e..0b92ce39c5b09 100644 --- a/crates/ruff/src/settings/defaults.rs +++ b/crates/ruff/src/settings/defaults.rs @@ -58,8 +58,14 @@ pub static EXCLUDE: Lazy> = Lazy::new(|| { ] }); -pub static INCLUDE: Lazy> = - Lazy::new(|| vec![FilePattern::Builtin("*.py"), FilePattern::Builtin("*.pyi")]); +pub static INCLUDE: Lazy> = Lazy::new(|| { + vec![ + FilePattern::Builtin("*.py"), + FilePattern::Builtin("*.pyi"), + FilePattern::Builtin("pyproject.toml"), + FilePattern::Builtin("*.toml"), + ] +}); impl Default for Settings { fn default() -> Self { diff --git a/crates/ruff_cli/src/diagnostics.rs b/crates/ruff_cli/src/diagnostics.rs index 4c77e73fbad73..fcc900b218fbd 100644 --- a/crates/ruff_cli/src/diagnostics.rs +++ b/crates/ruff_cli/src/diagnostics.rs @@ -18,9 +18,11 @@ use ruff::jupyter::{is_jupyter_notebook, JupyterIndex, JupyterNotebook}; use ruff::linter::{lint_fix, lint_only, FixTable, FixerResult, LinterResult}; use ruff::logging::DisplayParseError; use ruff::message::Message; +use ruff::pyproject_toml::lint_pyproject_toml; use ruff::settings::{flags, AllSettings, Settings}; use ruff_python_ast::imports::ImportMap; use ruff_python_ast::source_code::{LineIndex, SourceCode, SourceFileBuilder}; +use ruff_python_stdlib::path::is_project_toml; use crate::cache; @@ -130,6 +132,17 @@ pub(crate) fn lint_path( debug!("Checking: {}", path.display()); + // We have to special case this here since the python tokenizer doesn't work with toml + if is_project_toml(path) { + let messages = lint_pyproject_toml(path)?; + return Ok(Diagnostics { + messages, + fixed: FxHashMap::default(), + imports: ImportMap::default(), + jupyter_index: FxHashMap::default(), + }); + } + // Read the file from disk let (contents, jupyter_index) = if is_jupyter_notebook(path) { match load_jupyter_notebook(path) { diff --git a/crates/ruff_python_stdlib/src/path.rs b/crates/ruff_python_stdlib/src/path.rs index 2c48f44c2d4ec..733c18bb3fcdd 100644 --- a/crates/ruff_python_stdlib/src/path.rs +++ b/crates/ruff_python_stdlib/src/path.rs @@ -6,6 +6,12 @@ pub fn is_python_file(path: &Path) -> bool { .map_or(false, |ext| ext == "py" || ext == "pyi") } +/// Return `true` if the [`Path`] is named `pyproject.toml`. +pub fn is_project_toml(path: &Path) -> bool { + path.file_name() + .map_or(false, |name| name == "pyproject.toml") +} + /// Return `true` if the [`Path`] appears to be that of a Python interface definition file (`.pyi`). pub fn is_python_stub_file(path: &Path) -> bool { path.extension().map_or(false, |ext| ext == "pyi") From 1653ece84b01a4e84712b6f6558d561949002b0c Mon Sep 17 00:00:00 2001 From: konstin Date: Thu, 18 May 2023 16:53:18 +0200 Subject: [PATCH 02/12] Update schema --- ruff.schema.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ruff.schema.json b/ruff.schema.json index 1bcfb5fb08d67..0a47edb5c9114 100644 --- a/ruff.schema.json +++ b/ruff.schema.json @@ -2246,6 +2246,9 @@ "RUF1", "RUF10", "RUF100", + "RUF2", + "RUF20", + "RUF200", "S", "S1", "S10", From 825da4eef8e4e11e7810aa88bdc9d43880089126 Mon Sep 17 00:00:00 2001 From: konstin Date: Thu, 18 May 2023 17:10:20 +0200 Subject: [PATCH 03/12] Shorten path to fix windows tests --- crates/ruff/src/pyproject_toml.rs | 9 ++------- crates/ruff/src/rules/ruff/mod.rs | 7 ++++++- .../ruff__rules__ruff__tests__RUF200_bleach.snap | 2 +- .../ruff__rules__ruff__tests__RUF200_invalid_author.snap | 2 +- ...ruff__rules__ruff__tests__RUF200_maturin_gh_1615.snap | 2 +- crates/ruff_cli/src/diagnostics.rs | 4 +++- 6 files changed, 14 insertions(+), 12 deletions(-) diff --git a/crates/ruff/src/pyproject_toml.rs b/crates/ruff/src/pyproject_toml.rs index 19a1643747a30..90e2eddd2e344 100644 --- a/crates/ruff/src/pyproject_toml.rs +++ b/crates/ruff/src/pyproject_toml.rs @@ -1,19 +1,14 @@ -use std::fs; -use std::path::Path; - use anyhow::Result; use pyproject_toml::PyProjectToml; use ruff_text_size::{TextRange, TextSize}; use ruff_diagnostics::Diagnostic; -use ruff_python_ast::source_code::SourceFileBuilder; +use ruff_python_ast::source_code::SourceFile; use crate::message::Message; use crate::rules::ruff::rules::InvalidPyprojectToml; -pub fn lint_pyproject_toml(path: &Path) -> Result> { - let contents = fs::read_to_string(path)?; - let source_file = SourceFileBuilder::new(path.to_string_lossy(), contents).finish(); +pub fn lint_pyproject_toml(source_file: SourceFile) -> Result> { let err = match toml::from_str::(source_file.source_text()) { Ok(_) => return Ok(Vec::default()), Err(err) => err, diff --git a/crates/ruff/src/rules/ruff/mod.rs b/crates/ruff/src/rules/ruff/mod.rs index 8fae02be0f0c5..d888933e00fe8 100644 --- a/crates/ruff/src/rules/ruff/mod.rs +++ b/crates/ruff/src/rules/ruff/mod.rs @@ -4,12 +4,15 @@ pub(crate) mod rules; #[cfg(test)] mod tests { + use std::fs; use std::path::Path; use anyhow::Result; use rustc_hash::FxHashSet; use test_case::test_case; + use ruff_python_ast::source_code::SourceFileBuilder; + use crate::pyproject_toml::lint_pyproject_toml; use crate::registry::Rule; use crate::settings::resolve_per_file_ignores; @@ -187,7 +190,9 @@ mod tests { .join("pyproject_toml") .join(path) .join("pyproject.toml"); - let messages = lint_pyproject_toml(&path)?; + let contents = fs::read_to_string(path)?; + let source_file = SourceFileBuilder::new("pyproject.toml", contents).finish(); + let messages = lint_pyproject_toml(source_file)?; assert_messages!(snapshot, messages); Ok(()) } diff --git a/crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_bleach.snap b/crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_bleach.snap index f0fabf69b1b85..6b8d240bdb38b 100644 --- a/crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_bleach.snap +++ b/crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_bleach.snap @@ -1,7 +1,7 @@ --- source: crates/ruff/src/rules/ruff/mod.rs --- -./resources/test/fixtures/ruff/pyproject_toml/bleach/pyproject.toml:5:16: RUF200 Failed to parse pyproject.toml: Version specifier `>=1.1.0<1.2` doesn't match PEP 440 rules +pyproject.toml:5:16: RUF200 Failed to parse pyproject.toml: Version specifier `>=1.1.0<1.2` doesn't match PEP 440 rules tinycss2>=1.1.0<1.2 ^^^^^^^^^^^ | diff --git a/crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_invalid_author.snap b/crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_invalid_author.snap index 62d14287ac037..e58305050110b 100644 --- a/crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_invalid_author.snap +++ b/crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_invalid_author.snap @@ -1,7 +1,7 @@ --- source: crates/ruff/src/rules/ruff/mod.rs --- -./resources/test/fixtures/ruff/pyproject_toml/invalid_author/pyproject.toml:6:30: RUF200 Failed to parse pyproject.toml: invalid type: integer `1`, expected a string +pyproject.toml:6:30: RUF200 Failed to parse pyproject.toml: invalid type: integer `1`, expected a string | 6 | # Ensure that the spans from toml handle utf-8 correctly 7 | authors = [ diff --git a/crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_maturin_gh_1615.snap b/crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_maturin_gh_1615.snap index e136c2a10895d..b73fcf6cf745c 100644 --- a/crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_maturin_gh_1615.snap +++ b/crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_maturin_gh_1615.snap @@ -1,7 +1,7 @@ --- source: crates/ruff/src/rules/ruff/mod.rs --- -./resources/test/fixtures/ruff/pyproject_toml/maturin_gh_1615/pyproject.toml:9:17: RUF200 Failed to parse pyproject.toml: wanted string or table +pyproject.toml:9:17: RUF200 Failed to parse pyproject.toml: wanted string or table | 9 | [project] 10 | name = "..." diff --git a/crates/ruff_cli/src/diagnostics.rs b/crates/ruff_cli/src/diagnostics.rs index fcc900b218fbd..3786e1592cd38 100644 --- a/crates/ruff_cli/src/diagnostics.rs +++ b/crates/ruff_cli/src/diagnostics.rs @@ -134,7 +134,9 @@ pub(crate) fn lint_path( // We have to special case this here since the python tokenizer doesn't work with toml if is_project_toml(path) { - let messages = lint_pyproject_toml(path)?; + let contents = std::fs::read_to_string(path)?; + let source_file = SourceFileBuilder::new(path.to_string_lossy(), contents).finish(); + let messages = lint_pyproject_toml(source_file)?; return Ok(Diagnostics { messages, fixed: FxHashMap::default(), From d4b2e5a144cbe0f26d43742190850e26ba890263 Mon Sep 17 00:00:00 2001 From: konstin Date: Thu, 18 May 2023 17:11:04 +0200 Subject: [PATCH 04/12] Fix typo in fixture --- .../fixtures/ruff/pyproject_toml/maturin_gh_1615/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/ruff/resources/test/fixtures/ruff/pyproject_toml/maturin_gh_1615/pyproject.toml b/crates/ruff/resources/test/fixtures/ruff/pyproject_toml/maturin_gh_1615/pyproject.toml index f566feabd690d..14e8b567ea1b1 100644 --- a/crates/ruff/resources/test/fixtures/ruff/pyproject_toml/maturin_gh_1615/pyproject.toml +++ b/crates/ruff/resources/test/fixtures/ruff/pyproject_toml/maturin_gh_1615/pyproject.toml @@ -32,7 +32,7 @@ devel = [] [tool.maturin] include = [ "...",] bindings = "pyo3" -compatability = "manylinux2014" +compatibility = "manylinux2014" [tool.pytest.ini_options] testpaths = [ "...",] From cb6e79cad906164b771c12fdfabd2df2230e2d58 Mon Sep 17 00:00:00 2001 From: konstin Date: Thu, 18 May 2023 19:13:17 +0200 Subject: [PATCH 05/12] Fix default inclusion pattern --- crates/ruff/src/settings/defaults.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/ruff/src/settings/defaults.rs b/crates/ruff/src/settings/defaults.rs index 0b92ce39c5b09..be834aabfc123 100644 --- a/crates/ruff/src/settings/defaults.rs +++ b/crates/ruff/src/settings/defaults.rs @@ -62,8 +62,7 @@ pub static INCLUDE: Lazy> = Lazy::new(|| { vec![ FilePattern::Builtin("*.py"), FilePattern::Builtin("*.pyi"), - FilePattern::Builtin("pyproject.toml"), - FilePattern::Builtin("*.toml"), + FilePattern::Builtin("**/pyproject.toml"), ] }); From 96abd71a83dbda71c0f4a6c550480d0e675cc415 Mon Sep 17 00:00:00 2001 From: konstin Date: Fri, 19 May 2023 10:49:38 +0200 Subject: [PATCH 06/12] Update crates/ruff_cli/src/diagnostics.rs Co-authored-by: Micha Reiser --- crates/ruff_cli/src/diagnostics.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/ruff_cli/src/diagnostics.rs b/crates/ruff_cli/src/diagnostics.rs index 3786e1592cd38..dc59e450e2616 100644 --- a/crates/ruff_cli/src/diagnostics.rs +++ b/crates/ruff_cli/src/diagnostics.rs @@ -139,9 +139,7 @@ pub(crate) fn lint_path( let messages = lint_pyproject_toml(source_file)?; return Ok(Diagnostics { messages, - fixed: FxHashMap::default(), - imports: ImportMap::default(), - jupyter_index: FxHashMap::default(), + ..Diagnostics::default(), }); } From 63118fd4185d4044dd1ac1949a9303f335eaa336 Mon Sep 17 00:00:00 2001 From: konstin Date: Mon, 22 May 2023 10:09:26 +0200 Subject: [PATCH 07/12] Replace hangul by something that works with monospace --- .../ruff/pyproject_toml/invalid_author/pyproject.toml | 2 +- .../ruff__rules__ruff__tests__RUF200_invalid_author.snap | 6 +++--- crates/ruff_cli/src/diagnostics.rs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/ruff/resources/test/fixtures/ruff/pyproject_toml/invalid_author/pyproject.toml b/crates/ruff/resources/test/fixtures/ruff/pyproject_toml/invalid_author/pyproject.toml index 675ad7afe990e..94243f3807890 100644 --- a/crates/ruff/resources/test/fixtures/ruff/pyproject_toml/invalid_author/pyproject.toml +++ b/crates/ruff/resources/test/fixtures/ruff/pyproject_toml/invalid_author/pyproject.toml @@ -3,5 +3,5 @@ name = "hello-world" version = "0.1.0" # Ensure that the spans from toml handle utf-8 correctly authors = [ - { name = "콘스턴틴", email = 1 } + { name = "Z͑ͫ̓ͪ̂ͫ̽͏̴̙̤̞͉͚̯̞̠͍A̴̵̜̰͔ͫ͗͢L̠ͨͧͩ͘G̴̻͈͍͔̹̑͗̎̅͛́Ǫ̵̹̻̝̳͂̌̌͘", email = 1 } ] diff --git a/crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_invalid_author.snap b/crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_invalid_author.snap index e58305050110b..889257d1a6f54 100644 --- a/crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_invalid_author.snap +++ b/crates/ruff/src/rules/ruff/snapshots/ruff__rules__ruff__tests__RUF200_invalid_author.snap @@ -1,12 +1,12 @@ --- source: crates/ruff/src/rules/ruff/mod.rs --- -pyproject.toml:6:30: RUF200 Failed to parse pyproject.toml: invalid type: integer `1`, expected a string +pyproject.toml:6:84: RUF200 Failed to parse pyproject.toml: invalid type: integer `1`, expected a string | 6 | # Ensure that the spans from toml handle utf-8 correctly 7 | authors = [ -8 | { name = "콘스턴틴", email = 1 } - | ^ RUF200 +8 | { name = "Z͑ͫ̓ͪ̂ͫ̽͏̴̙̤̞͉͚̯̞̠͍A̴̵̜̰͔ͫ͗͢L̠ͨͧͩ͘G̴̻͈͍͔̹̑͗̎̅͛́Ǫ̵̹̻̝̳͂̌̌͘", email = 1 } + | ^ RUF200 9 | ] | diff --git a/crates/ruff_cli/src/diagnostics.rs b/crates/ruff_cli/src/diagnostics.rs index dc59e450e2616..1cf5268c88b18 100644 --- a/crates/ruff_cli/src/diagnostics.rs +++ b/crates/ruff_cli/src/diagnostics.rs @@ -139,7 +139,7 @@ pub(crate) fn lint_path( let messages = lint_pyproject_toml(source_file)?; return Ok(Diagnostics { messages, - ..Diagnostics::default(), + ..Diagnostics::default() }); } From 1d67d964409a54001eb46aba285adbda2cdffe41 Mon Sep 17 00:00:00 2001 From: konstin Date: Mon, 22 May 2023 10:12:57 +0200 Subject: [PATCH 08/12] Link missing span issue --- crates/ruff/src/pyproject_toml.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/ruff/src/pyproject_toml.rs b/crates/ruff/src/pyproject_toml.rs index 90e2eddd2e344..b82875f88081c 100644 --- a/crates/ruff/src/pyproject_toml.rs +++ b/crates/ruff/src/pyproject_toml.rs @@ -16,6 +16,7 @@ pub fn lint_pyproject_toml(source_file: SourceFile) -> Result> { let range = match err.span() { // This is bad but sometimes toml and/or serde just don't give us spans + // TODO(konstin,micha): https://github.com/charliermarsh/ruff/issues/4571 None => TextRange::default(), Some(range) => { let expect_err = "pyproject.toml file be smaller than 4GB"; From 6ec6e03a7f6c8b579efb18bfb0afba727a86dc98 Mon Sep 17 00:00:00 2001 From: konstin Date: Mon, 22 May 2023 10:17:03 +0200 Subject: [PATCH 09/12] Don't panic on too large files --- crates/ruff/src/pyproject_toml.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/ruff/src/pyproject_toml.rs b/crates/ruff/src/pyproject_toml.rs index b82875f88081c..8f10b985fbe66 100644 --- a/crates/ruff/src/pyproject_toml.rs +++ b/crates/ruff/src/pyproject_toml.rs @@ -1,4 +1,4 @@ -use anyhow::Result; +use anyhow::{Context, Result}; use pyproject_toml::PyProjectToml; use ruff_text_size::{TextRange, TextSize}; @@ -19,10 +19,10 @@ pub fn lint_pyproject_toml(source_file: SourceFile) -> Result> { // TODO(konstin,micha): https://github.com/charliermarsh/ruff/issues/4571 None => TextRange::default(), Some(range) => { - let expect_err = "pyproject.toml file be smaller than 4GB"; + let expect_err = "pyproject.toml should be smaller than 4GB"; TextRange::new( - TextSize::try_from(range.start).expect(expect_err), - TextSize::try_from(range.end).expect(expect_err), + TextSize::try_from(range.start).context(expect_err)?, + TextSize::try_from(range.end).context(expect_err)?, ) } }; From 477cf657d67ea22d94f5942360162027b6b5dfe4 Mon Sep 17 00:00:00 2001 From: konstin Date: Mon, 22 May 2023 10:23:40 +0200 Subject: [PATCH 10/12] Emit diagnostic for pyproject.toml > 4GB --- crates/ruff/src/pyproject_toml.rs | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/crates/ruff/src/pyproject_toml.rs b/crates/ruff/src/pyproject_toml.rs index 8f10b985fbe66..a91f898326431 100644 --- a/crates/ruff/src/pyproject_toml.rs +++ b/crates/ruff/src/pyproject_toml.rs @@ -1,4 +1,4 @@ -use anyhow::{Context, Result}; +use anyhow::Result; use pyproject_toml::PyProjectToml; use ruff_text_size::{TextRange, TextSize}; @@ -7,6 +7,7 @@ use ruff_python_ast::source_code::SourceFile; use crate::message::Message; use crate::rules::ruff::rules::InvalidPyprojectToml; +use crate::IOError; pub fn lint_pyproject_toml(source_file: SourceFile) -> Result> { let err = match toml::from_str::(source_file.source_text()) { @@ -19,10 +20,26 @@ pub fn lint_pyproject_toml(source_file: SourceFile) -> Result> { // TODO(konstin,micha): https://github.com/charliermarsh/ruff/issues/4571 None => TextRange::default(), Some(range) => { - let expect_err = "pyproject.toml should be smaller than 4GB"; + let end = match TextSize::try_from(range.end) { + Ok(end) => end, + Err(_) => { + let diagnostic = Diagnostic::new( + IOError { + message: format!("pyproject.toml is larger than 4GB"), + }, + TextRange::default(), + ); + return Ok(vec![Message::from_diagnostic( + diagnostic, + source_file, + TextSize::default(), + )]); + } + }; TextRange::new( - TextSize::try_from(range.start).context(expect_err)?, - TextSize::try_from(range.end).context(expect_err)?, + // start <= end, so if end < 4GB follows start < 4GB + TextSize::try_from(range.start).unwrap(), + end, ) } }; From 1a059015a52cf5e13145cba96bb6e03d59817017 Mon Sep 17 00:00:00 2001 From: konstin Date: Wed, 24 May 2023 16:47:31 +0200 Subject: [PATCH 11/12] Make build system optional --- crates/ruff/src/pyproject_toml.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/crates/ruff/src/pyproject_toml.rs b/crates/ruff/src/pyproject_toml.rs index a91f898326431..0237632e90846 100644 --- a/crates/ruff/src/pyproject_toml.rs +++ b/crates/ruff/src/pyproject_toml.rs @@ -1,6 +1,7 @@ use anyhow::Result; -use pyproject_toml::PyProjectToml; +use pyproject_toml::{BuildSystem, Project}; use ruff_text_size::{TextRange, TextSize}; +use serde::{Deserialize, Serialize}; use ruff_diagnostics::Diagnostic; use ruff_python_ast::source_code::SourceFile; @@ -9,6 +10,16 @@ use crate::message::Message; use crate::rules::ruff::rules::InvalidPyprojectToml; use crate::IOError; +/// Unlike [pyproject_toml::PyProjectToml], in our case `build_system` is also optional +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +struct PyProjectToml { + /// Build-related data + build_system: Option, + /// Project metadata + project: Option, +} + pub fn lint_pyproject_toml(source_file: SourceFile) -> Result> { let err = match toml::from_str::(source_file.source_text()) { Ok(_) => return Ok(Vec::default()), From 5ae314dc21f05f4a5388f5a087d3c4e6e3ce31df Mon Sep 17 00:00:00 2001 From: konstin Date: Thu, 25 May 2023 13:49:39 +0200 Subject: [PATCH 12/12] cargo clippy --- crates/ruff/src/pyproject_toml.rs | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/crates/ruff/src/pyproject_toml.rs b/crates/ruff/src/pyproject_toml.rs index 0237632e90846..21d3c04d1afe7 100644 --- a/crates/ruff/src/pyproject_toml.rs +++ b/crates/ruff/src/pyproject_toml.rs @@ -10,7 +10,7 @@ use crate::message::Message; use crate::rules::ruff::rules::InvalidPyprojectToml; use crate::IOError; -/// Unlike [pyproject_toml::PyProjectToml], in our case `build_system` is also optional +/// Unlike [`pyproject_toml::PyProjectToml`], in our case `build_system` is also optional #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] struct PyProjectToml { @@ -31,21 +31,18 @@ pub fn lint_pyproject_toml(source_file: SourceFile) -> Result> { // TODO(konstin,micha): https://github.com/charliermarsh/ruff/issues/4571 None => TextRange::default(), Some(range) => { - let end = match TextSize::try_from(range.end) { - Ok(end) => end, - Err(_) => { - let diagnostic = Diagnostic::new( - IOError { - message: format!("pyproject.toml is larger than 4GB"), - }, - TextRange::default(), - ); - return Ok(vec![Message::from_diagnostic( - diagnostic, - source_file, - TextSize::default(), - )]); - } + let Ok(end) = TextSize::try_from(range.end) else { + let diagnostic = Diagnostic::new( + IOError { + message: "pyproject.toml is larger than 4GB".to_string(), + }, + TextRange::default(), + ); + return Ok(vec![Message::from_diagnostic( + diagnostic, + source_file, + TextSize::default(), + )]); }; TextRange::new( // start <= end, so if end < 4GB follows start < 4GB