From 7833ee7dd083c10f097295cda96c752525723f75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Rodr=C3=ADguez?= Date: Sun, 4 Sep 2022 13:44:20 +0200 Subject: [PATCH 01/10] Remove code dead It's no longer used since 21154847711101f92412ae4d0b69cfab7a3044ac. --- python/lib/dependabot/python/file_parser.rb | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index 0aa6c1e750c..6c96bf71c44 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -105,18 +105,6 @@ def group_from_filename(filename) end end - def included_in_pipenv_deps?(dep_name) - return false unless pipfile - - pipenv_dependencies.dependencies.map(&:name).include?(dep_name) - end - - def included_in_poetry_deps?(dep_name) - return false unless using_poetry? - - poetry_dependencies.dependencies.map(&:name).include?(dep_name) - end - def blocking_marker?(dep) return false if dep["markers"] == "None" return true if dep["markers"].include?("<") From 29cd3bbe655b0573a57eed885cb995d5ac617234 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Rodr=C3=ADguez?= Date: Sun, 4 Sep 2022 13:52:59 +0200 Subject: [PATCH 02/10] Remove unused constant It's unused since it was first introduced at 65ee6f80c4dec02be546a7526fc5317c0233b647, since it's duplicated in `PoetryFilesParser`. --- python/lib/dependabot/python/file_parser.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index 6c96bf71c44..87898489614 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -18,8 +18,6 @@ class FileParser < Dependabot::FileParsers::Base require_relative "file_parser/poetry_files_parser" require_relative "file_parser/setup_file_parser" - POETRY_DEPENDENCY_TYPES = - %w(tool.poetry.dependencies tool.poetry.dev-dependencies).freeze DEPENDENCY_GROUP_KEYS = [ { pipfile: "packages", From 9de08201defb1a7616788cdb685b4784ccb6c116 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Rodr=C3=ADguez?= Date: Thu, 15 Sep 2022 17:22:29 +0200 Subject: [PATCH 03/10] Some renaming and rewording To make room for standard python specs and clarify what fixture manifests test. --- ...spec.rb => pyproject_files_parser_spec.rb} | 21 ++++++++++--------- .../dependabot/python/file_parser_spec.rb | 6 +++--- .../file_updater/pipfile_file_updater_spec.rb | 2 +- .../file_updater/pyproject_preparer_spec.rb | 6 +++--- .../pipenv_version_resolver_spec.rb | 2 +- ...ct.toml => basic_poetry_dependencies.toml} | 0 .../{pyproject.lock => poetry.lock} | 0 7 files changed, 19 insertions(+), 18 deletions(-) rename python/spec/dependabot/python/file_parser/{poetry_files_parser_spec.rb => pyproject_files_parser_spec.rb} (93%) rename python/spec/fixtures/pyproject_files/{pyproject.toml => basic_poetry_dependencies.toml} (100%) rename python/spec/fixtures/pyproject_locks/{pyproject.lock => poetry.lock} (100%) diff --git a/python/spec/dependabot/python/file_parser/poetry_files_parser_spec.rb b/python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb similarity index 93% rename from python/spec/dependabot/python/file_parser/poetry_files_parser_spec.rb rename to python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb index b0d3bf6ee7e..71e717d7b91 100644 --- a/python/spec/dependabot/python/file_parser/poetry_files_parser_spec.rb +++ b/python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb @@ -17,9 +17,10 @@ let(:pyproject_body) do fixture("pyproject_files", pyproject_fixture_name) end - let(:pyproject_fixture_name) { "pyproject.toml" } - describe "parse" do + describe "parse poetry files" do + let(:pyproject_fixture_name) { "basic_poetry_dependencies.toml" } + subject(:dependencies) { parser.dependency_set.dependencies } context "without a lockfile" do @@ -102,17 +103,17 @@ end context "with a lockfile" do - let(:files) { [pyproject, pyproject_lock] } - let(:pyproject_lock) do + let(:files) { [pyproject, poetry_lock] } + let(:poetry_lock) do Dependabot::DependencyFile.new( - name: "pyproject.lock", + name: "poetry.lock", content: pyproject_lock_body ) end let(:pyproject_lock_body) do fixture("pyproject_locks", pyproject_lock_fixture_name) end - let(:pyproject_lock_fixture_name) { "pyproject.lock" } + let(:pyproject_lock_fixture_name) { "poetry.lock" } its(:length) { is_expected.to eq(36) } @@ -120,11 +121,11 @@ expect(dependencies.map(&:name)).to_not include("python") end - context "that is called poetry.lock" do - let(:files) { [pyproject, poetry_lock] } - let(:poetry_lock) do + context "that is called pyproject.lock (legacy name)" do + let(:files) { [pyproject, pyproject_lock] } + let(:pyproject_lock) do Dependabot::DependencyFile.new( - name: "poetry.lock", + name: "pyproject.lock", content: pyproject_lock_body ) end diff --git a/python/spec/dependabot/python/file_parser_spec.rb b/python/spec/dependabot/python/file_parser_spec.rb index fb944a54dd5..81fbb1088b6 100644 --- a/python/spec/dependabot/python/file_parser_spec.rb +++ b/python/spec/dependabot/python/file_parser_spec.rb @@ -1212,18 +1212,18 @@ end end - context "with a pyproject.toml and pyproject.lock" do + context "with a pyproject.toml in poetry format and pyproject.lock legacy poetry lock file" do let(:files) { [pyproject, pyproject_lock] } let(:pyproject) do Dependabot::DependencyFile.new( name: "pyproject.toml", - content: fixture("pyproject_files", "pyproject.toml") + content: fixture("pyproject_files", "basic_poetry_dependencies.toml") ) end let(:pyproject_lock) do Dependabot::DependencyFile.new( name: "pyproject.lock", - content: fixture("pyproject_locks", "pyproject.lock") + content: fixture("pyproject_locks", "poetry.lock") ) end diff --git a/python/spec/dependabot/python/file_updater/pipfile_file_updater_spec.rb b/python/spec/dependabot/python/file_updater/pipfile_file_updater_spec.rb index fae5cc0b6ba..345d2120886 100644 --- a/python/spec/dependabot/python/file_updater/pipfile_file_updater_spec.rb +++ b/python/spec/dependabot/python/file_updater/pipfile_file_updater_spec.rb @@ -155,7 +155,7 @@ let(:pyproject) do Dependabot::DependencyFile.new( name: "pyproject.toml", - content: fixture("pyproject_files", "pyproject.toml") + content: fixture("pyproject_files", "basic_poetry_dependencies.toml") ) end diff --git a/python/spec/dependabot/python/file_updater/pyproject_preparer_spec.rb b/python/spec/dependabot/python/file_updater/pyproject_preparer_spec.rb index f48fd41a648..656020cea0c 100644 --- a/python/spec/dependabot/python/file_updater/pyproject_preparer_spec.rb +++ b/python/spec/dependabot/python/file_updater/pyproject_preparer_spec.rb @@ -14,7 +14,7 @@ end let(:lockfile) { nil } let(:pyproject_content) { fixture("pyproject_files", pyproject_fixture_name) } - let(:pyproject_fixture_name) { "pyproject.toml" } + let(:pyproject_fixture_name) { "basic_poetry_dependencies.toml" } describe "#add_auth_env_vars" do it "adds auth env vars when a token is present" do @@ -48,7 +48,7 @@ it "doesn't break when there are no private sources" do preparer = Dependabot::Python::FileUpdater::PyprojectPreparer.new( - pyproject_content: fixture("pyproject_files", "pyproject.toml"), + pyproject_content: pyproject_content, lockfile: nil ) expect { preparer.add_auth_env_vars(nil) }.not_to raise_error @@ -99,7 +99,7 @@ let(:pyproject_lock_body) do fixture("pyproject_locks", pyproject_lock_fixture_name) end - let(:pyproject_lock_fixture_name) { "pyproject.lock" } + let(:pyproject_lock_fixture_name) { "poetry.lock" } context "with no dependencies to except" do let(:dependencies) { [] } diff --git a/python/spec/dependabot/python/update_checker/pipenv_version_resolver_spec.rb b/python/spec/dependabot/python/update_checker/pipenv_version_resolver_spec.rb index aec3147ccc5..1c0afacba90 100644 --- a/python/spec/dependabot/python/update_checker/pipenv_version_resolver_spec.rb +++ b/python/spec/dependabot/python/update_checker/pipenv_version_resolver_spec.rb @@ -188,7 +188,7 @@ let(:pyproject) do Dependabot::DependencyFile.new( name: "pyproject.toml", - content: fixture("pyproject_files", "pyproject.toml") + content: fixture("pyproject_files", "basic_poetry_dependencies.toml") ) end diff --git a/python/spec/fixtures/pyproject_files/pyproject.toml b/python/spec/fixtures/pyproject_files/basic_poetry_dependencies.toml similarity index 100% rename from python/spec/fixtures/pyproject_files/pyproject.toml rename to python/spec/fixtures/pyproject_files/basic_poetry_dependencies.toml diff --git a/python/spec/fixtures/pyproject_locks/pyproject.lock b/python/spec/fixtures/pyproject_locks/poetry.lock similarity index 100% rename from python/spec/fixtures/pyproject_locks/pyproject.lock rename to python/spec/fixtures/pyproject_locks/poetry.lock From b82036265cb32a9b9b0b03895a5b5f5a5237f46a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Rodr=C3=ADguez?= Date: Thu, 29 Sep 2022 13:13:49 +0200 Subject: [PATCH 04/10] Rename another fixture file to clarify it has poetry reqs --- .../spec/dependabot/python/update_checker/index_finder_spec.rb | 2 +- .../python/update_checker/latest_version_finder_spec.rb | 2 +- .../python/update_checker/poetry_version_resolver_spec.rb | 2 +- python/spec/dependabot/python/update_checker_spec.rb | 2 +- .../{exact_version.toml => poetry_exact_requirement.toml} | 0 5 files changed, 4 insertions(+), 4 deletions(-) rename python/spec/fixtures/pyproject_files/{exact_version.toml => poetry_exact_requirement.toml} (100%) diff --git a/python/spec/dependabot/python/update_checker/index_finder_spec.rb b/python/spec/dependabot/python/update_checker/index_finder_spec.rb index 0f3abb76881..8c82838e420 100644 --- a/python/spec/dependabot/python/update_checker/index_finder_spec.rb +++ b/python/spec/dependabot/python/update_checker/index_finder_spec.rb @@ -40,7 +40,7 @@ content: fixture("pyproject_files", pyproject_fixture_name) ) end - let(:pyproject_fixture_name) { "exact_version.toml" } + let(:pyproject_fixture_name) { "poetry_exact_requirement.toml" } let(:requirements_file) do Dependabot::DependencyFile.new( name: "requirements.txt", diff --git a/python/spec/dependabot/python/update_checker/latest_version_finder_spec.rb b/python/spec/dependabot/python/update_checker/latest_version_finder_spec.rb index 29dcc78c24a..a49f0450fcc 100644 --- a/python/spec/dependabot/python/update_checker/latest_version_finder_spec.rb +++ b/python/spec/dependabot/python/update_checker/latest_version_finder_spec.rb @@ -48,7 +48,7 @@ content: fixture("pyproject_files", pyproject_fixture_name) ) end - let(:pyproject_fixture_name) { "exact_version.toml" } + let(:pyproject_fixture_name) { "poetry_exact_requirement.toml" } let(:requirements_file) do Dependabot::DependencyFile.new( name: "requirements.txt", diff --git a/python/spec/dependabot/python/update_checker/poetry_version_resolver_spec.rb b/python/spec/dependabot/python/update_checker/poetry_version_resolver_spec.rb index 34a76353c38..bfd80b7ed4b 100644 --- a/python/spec/dependabot/python/update_checker/poetry_version_resolver_spec.rb +++ b/python/spec/dependabot/python/update_checker/poetry_version_resolver_spec.rb @@ -29,7 +29,7 @@ content: fixture("pyproject_files", pyproject_fixture_name) ) end - let(:pyproject_fixture_name) { "exact_version.toml" } + let(:pyproject_fixture_name) { "poetry_exact_requirement.toml" } let(:lockfile) do Dependabot::DependencyFile.new( name: "pyproject.lock", diff --git a/python/spec/dependabot/python/update_checker_spec.rb b/python/spec/dependabot/python/update_checker_spec.rb index 50590b1c039..37afb3e0391 100644 --- a/python/spec/dependabot/python/update_checker_spec.rb +++ b/python/spec/dependabot/python/update_checker_spec.rb @@ -49,7 +49,7 @@ content: fixture("pyproject_files", pyproject_fixture_name) ) end - let(:pyproject_fixture_name) { "exact_version.toml" } + let(:pyproject_fixture_name) { "poetry_exact_requirement.toml" } let(:requirements_file) do Dependabot::DependencyFile.new( name: "requirements.txt", diff --git a/python/spec/fixtures/pyproject_files/exact_version.toml b/python/spec/fixtures/pyproject_files/poetry_exact_requirement.toml similarity index 100% rename from python/spec/fixtures/pyproject_files/exact_version.toml rename to python/spec/fixtures/pyproject_files/poetry_exact_requirement.toml From e3592bb05f8369bed56fdd47cf340aa072656be0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Rodr=C3=ADguez?= Date: Thu, 1 Sep 2022 10:52:24 +0200 Subject: [PATCH 05/10] Add Pep621 pyproject.toml format support --- python/helpers/lib/parser.py | 33 ++++++++++ python/helpers/run.py | 2 + python/lib/dependabot/python/file_fetcher.rb | 10 +-- python/lib/dependabot/python/file_parser.rb | 20 ++---- ...es_parser.rb => pyproject_files_parser.rb} | 65 ++++++++++++++++++- python/lib/dependabot/python/file_updater.rb | 15 ++++- .../file_updater/poetry_file_updater.rb | 2 +- .../python/file_updater/pyproject_preparer.rb | 2 +- .../lib/dependabot/python/update_checker.rb | 25 +++++-- .../update_checker/poetry_version_resolver.rb | 2 +- .../pyproject_files_parser_spec.rb | 30 ++++++++- .../dependabot/python/file_updater_spec.rb | 39 +++++++++++ .../dependabot/python/update_checker_spec.rb | 53 +++++++++++---- .../pep621_exact_requirement.toml | 13 ++++ .../pyproject_files/standard_python.toml | 13 ++++ 15 files changed, 278 insertions(+), 46 deletions(-) rename python/lib/dependabot/python/file_parser/{poetry_files_parser.rb => pyproject_files_parser.rb} (72%) create mode 100644 python/spec/fixtures/pyproject_files/pep621_exact_requirement.toml create mode 100644 python/spec/fixtures/pyproject_files/standard_python.toml diff --git a/python/helpers/lib/parser.py b/python/helpers/lib/parser.py index b167318db4e..c9185386952 100644 --- a/python/helpers/lib/parser.py +++ b/python/helpers/lib/parser.py @@ -11,11 +11,44 @@ install_req_from_line, install_req_from_parsed_requirement, ) + +from packaging.requirements import InvalidRequirement, Requirement +import toml + # Inspired by pips internal check: # https://github.com/pypa/pip/blob/0bb3ac87f5bb149bd75cceac000844128b574385/src/pip/_internal/req/req_file.py#L35 COMMENT_RE = re.compile(r'(^|\s+)#.*$') +def parse_pep621_dependencies(pyproject_path): + dependencies = toml.load(pyproject_path)['project']['dependencies'] + + requirement_packages = [] + + def version_from_req(specifier_set): + if (len(specifier_set) == 1 and + next(iter(specifier_set)).operator in {"==", "==="}): + return next(iter(specifier_set)).version + + for dependency in dependencies: + try: + req = Requirement(dependency) + except InvalidRequirement as e: + print(json.dumps({"error": repr(e)})) + exit(1) + else: + requirement_packages.append({ + "name": req.name, + "version": version_from_req(req.specifier), + "markers": str(req.marker) or None, + "file": pyproject_path, + "requirement": str(req.specifier), + "extras": sorted(list(req.extras)) + }) + + return json.dumps({"result": requirement_packages}) + + def parse_requirements(directory): # Parse the requirements.txt requirement_packages = [] diff --git a/python/helpers/run.py b/python/helpers/run.py index 26961e502d2..f80f8ff2ec5 100644 --- a/python/helpers/run.py +++ b/python/helpers/run.py @@ -10,6 +10,8 @@ print(parser.parse_requirements(args["args"][0])) elif args["function"] == "parse_setup": print(parser.parse_setup(args["args"][0])) + elif args["function"] == "parse_pep621_dependencies": + print(parser.parse_pep621_dependencies(args["args"][0])) elif args["function"] == "get_dependency_hash": print(hasher.get_dependency_hash(*args["args"])) elif args["function"] == "get_pipfile_hash": diff --git a/python/lib/dependabot/python/file_fetcher.rb b/python/lib/dependabot/python/file_fetcher.rb index 5a2aea67c47..1318c875d5a 100644 --- a/python/lib/dependabot/python/file_fetcher.rb +++ b/python/lib/dependabot/python/file_fetcher.rb @@ -5,7 +5,7 @@ require "dependabot/file_fetchers" require "dependabot/file_fetchers/base" require "dependabot/python/requirement_parser" -require "dependabot/python/file_parser/poetry_files_parser" +require "dependabot/python/file_parser/pyproject_files_parser" require "dependabot/errors" module Dependabot @@ -24,7 +24,7 @@ def self.required_files_in?(filenames) # If this repo is using a Pipfile return true return true if filenames.include?("Pipfile") - # If this repo is using Poetry return true + # If this repo is using pyproject.toml return true return true if filenames.include?("pyproject.toml") return true if filenames.include?("setup.py") @@ -296,8 +296,8 @@ def fetch_path_setup_file(path, allow_pyproject: false) fetch_submodules: true ).tap { |f| f.support_file = true } rescue Dependabot::DependencyFileNotFound - # For Poetry projects attempt to fetch a pyproject.toml at the - # given path instead of a setup.py. We do not require a + # For projects with pyproject.toml attempt to fetch a pyproject.toml + # at the given path instead of a setup.py. We do not require a # setup.py to be present, so if none can be found, simply return return [] unless allow_pyproject @@ -395,7 +395,7 @@ def poetry_path_setup_file_paths return [] unless pyproject paths = [] - Dependabot::Python::FileParser::PoetryFilesParser::POETRY_DEPENDENCY_TYPES.each do |dep_type| + Dependabot::Python::FileParser::PyprojectFilesParser::POETRY_DEPENDENCY_TYPES.each do |dep_type| next unless parsed_pyproject.dig("tool", "poetry", dep_type) parsed_pyproject.dig("tool", "poetry", dep_type).each do |_, req| diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index 87898489614..8ecc641763a 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true -require "toml-rb" require "dependabot/dependency" require "dependabot/file_parsers" require "dependabot/file_parsers/base" @@ -15,7 +14,7 @@ module Dependabot module Python class FileParser < Dependabot::FileParsers::Base require_relative "file_parser/pipfile_files_parser" - require_relative "file_parser/poetry_files_parser" + require_relative "file_parser/pyproject_files_parser" require_relative "file_parser/setup_file_parser" DEPENDENCY_GROUP_KEYS = [ @@ -40,7 +39,7 @@ def parse dependency_set = DependencySet.new dependency_set += pipenv_dependencies if pipfile - dependency_set += poetry_dependencies if using_poetry? + dependency_set += pyproject_file_dependencies if pyproject dependency_set += requirement_dependencies if requirement_files.any? dependency_set += setup_file_dependencies if setup_file || setup_cfg_file @@ -60,9 +59,9 @@ def pipenv_dependencies dependency_set end - def poetry_dependencies - @poetry_dependencies ||= - PoetryFilesParser. + def pyproject_file_dependencies + @pyproject_file_dependencies ||= + PyprojectFilesParser. new(dependency_files: dependency_files). dependency_set end @@ -201,15 +200,6 @@ def pipfile_lock @pipfile_lock ||= get_original_file("Pipfile.lock") end - def using_poetry? - return false unless pyproject - return true if poetry_lock || pyproject_lock - - !TomlRB.parse(pyproject.content).dig("tool", "poetry").nil? - rescue TomlRB::ParseError, TomlRB::ValueOverwriteError - raise Dependabot::DependencyFileNotParseable, pyproject.path - end - def output_file_regex(filename) "--output-file[=\s]+#{Regexp.escape(filename)}(?:\s|$)" end diff --git a/python/lib/dependabot/python/file_parser/poetry_files_parser.rb b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb similarity index 72% rename from python/lib/dependabot/python/file_parser/poetry_files_parser.rb rename to python/lib/dependabot/python/file_parser/pyproject_files_parser.rb index f604718f8f6..3b77de186e6 100644 --- a/python/lib/dependabot/python/file_parser/poetry_files_parser.rb +++ b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb @@ -12,7 +12,7 @@ module Dependabot module Python class FileParser - class PoetryFilesParser + class PyprojectFilesParser POETRY_DEPENDENCY_TYPES = %w(dependencies dev-dependencies).freeze # https://python-poetry.org/docs/dependency-specification/ @@ -36,6 +36,14 @@ def dependency_set attr_reader :dependency_files def pyproject_dependencies + if using_poetry? + poetry_dependencies + elsif using_pep621? + pep621_dependencies + end + end + + def poetry_dependencies dependencies = Dependabot::FileParsers::Base::DependencySet.new POETRY_DEPENDENCY_TYPES.each do |type| @@ -59,6 +67,35 @@ def pyproject_dependencies dependencies end + def pep621_dependencies + dependencies = Dependabot::FileParsers::Base::DependencySet.new + + parsed_pep621_dependencies.each do |dep| + # If a requirement has a `<` or `<=` marker then updating it is + # probably blocked. Ignore it. + next if dep["markers"].include?("<") + + dependencies << + Dependency.new( + name: normalised_name(dep["name"], dep["extras"]), + version: dep["version"]&.include?("*") ? nil : dep["version"], + requirements: [{ + requirement: dep["requirement"], + file: Pathname.new(dep["file"]).cleanpath.to_path, + source: nil, + groups: [dep["requirement_type"]] + }], + package_manager: "pip" + ) + end + + dependencies + end + + def normalised_name(name, extras) + NameNormaliser.normalise_including_extras(name, extras) + end + # @param req can be an Array, Hash or String that represents the constraints for a dependency def parse_requirements_from(req, type) [req].flatten.compact.filter_map do |requirement| @@ -75,6 +112,14 @@ def parse_requirements_from(req, type) end end + def using_poetry? + !parsed_pyproject.dig("tool", "poetry").nil? + end + + def using_pep621? + !parsed_pyproject.dig("project", "dependencies").nil? + end + # Create a DependencySet where each element has no requirement. Any # requirements will be added when combining the DependencySet with # other DependencySets. @@ -146,6 +191,24 @@ def lockfile poetry_lock || pyproject_lock end + def parsed_pep621_dependencies + SharedHelpers.in_a_temporary_directory do + write_temporary_pyproject + + SharedHelpers.run_helper_subprocess( + command: "pyenv exec python #{NativeHelpers.python_helper_path}", + function: "parse_pep621_dependencies", + args: [pyproject.name] + ) + end + end + + def write_temporary_pyproject + path = pyproject.name + FileUtils.mkdir_p(Pathname.new(path).dirname) + File.write(path, pyproject.content) + end + def parsed_lockfile return parsed_poetry_lock if poetry_lock return parsed_pyproject_lock if pyproject_lock diff --git a/python/lib/dependabot/python/file_updater.rb b/python/lib/dependabot/python/file_updater.rb index 75414ec0a96..94cea9f6070 100644 --- a/python/lib/dependabot/python/file_updater.rb +++ b/python/lib/dependabot/python/file_updater.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +require "toml-rb" require "dependabot/file_updaters" require "dependabot/file_updaters/base" require "dependabot/shared_helpers" @@ -61,7 +62,13 @@ def resolver_type # Otherwise, this is a top-level dependency, and we can figure out # which resolver to use based on the filename of its requirements return :pipfile if changed_req_files.any?("Pipfile") - return :poetry if changed_req_files.any?("pyproject.toml") + + if changed_req_files.any?("pyproject.toml") + return :poetry if poetry_based? + + return :requirements + end + return :pip_compile if changed_req_files.any? { |f| f.end_with?(".in") } :requirements @@ -119,6 +126,12 @@ def check_required_files raise "Missing required files!" end + def poetry_based? + return false unless pyproject + + !TomlRB.parse(pyproject.content).dig("tool", "poetry").nil? + end + def pipfile @pipfile ||= get_original_file("Pipfile") end diff --git a/python/lib/dependabot/python/file_updater/poetry_file_updater.rb b/python/lib/dependabot/python/file_updater/poetry_file_updater.rb index 9b53ac74e45..1fba530ea2f 100644 --- a/python/lib/dependabot/python/file_updater/poetry_file_updater.rb +++ b/python/lib/dependabot/python/file_updater/poetry_file_updater.rb @@ -131,7 +131,7 @@ def freeze_dependencies_being_updated(pyproject_content) end def lock_declaration_to_new_version!(poetry_object, dep) - Dependabot::Python::FileParser::PoetryFilesParser::POETRY_DEPENDENCY_TYPES.each do |type| + Dependabot::Python::FileParser::PyprojectFilesParser::POETRY_DEPENDENCY_TYPES.each do |type| names = poetry_object[type]&.keys || [] pkg_name = names.find { |nm| normalise(nm) == dep.name } next unless pkg_name diff --git a/python/lib/dependabot/python/file_updater/pyproject_preparer.rb b/python/lib/dependabot/python/file_updater/pyproject_preparer.rb index 72fd1649664..08fca9222b7 100644 --- a/python/lib/dependabot/python/file_updater/pyproject_preparer.rb +++ b/python/lib/dependabot/python/file_updater/pyproject_preparer.rb @@ -52,7 +52,7 @@ def freeze_top_level_dependencies_except(dependencies) poetry_object = pyproject_object["tool"]["poetry"] excluded_names = dependencies.map(&:name) + ["python"] - Dependabot::Python::FileParser::PoetryFilesParser::POETRY_DEPENDENCY_TYPES.each do |key| + Dependabot::Python::FileParser::PyprojectFilesParser::POETRY_DEPENDENCY_TYPES.each do |key| next unless poetry_object[key] source_types = %w(directory file url) diff --git a/python/lib/dependabot/python/update_checker.rb b/python/lib/dependabot/python/update_checker.rb index 91628470684..0b0700cd615 100644 --- a/python/lib/dependabot/python/update_checker.rb +++ b/python/lib/dependabot/python/update_checker.rb @@ -143,7 +143,7 @@ def resolver_type # Otherwise, this is a top-level dependency, and we can figure out # which resolver to use based on the filename of its requirements return :pipenv if updating_pipfile? - return :poetry if updating_pyproject? + return pyproject_resolver if updating_pyproject? return :pip_compile if updating_in_file? if dependency.version && !exact_requirement?(reqs) @@ -161,6 +161,12 @@ def subdependency_resolver raise "Claimed to be a sub-dependency, but no lockfile exists!" end + def pyproject_resolver + return :poetry if poetry_based? + + :requirements + end + def exact_requirement?(reqs) reqs = reqs.map { |r| r.fetch(:requirement) } reqs = reqs.compact @@ -258,22 +264,23 @@ def latest_version_finder ) end + def poetry_based? + updating_pyproject? && !poetry_details.nil? + end + def poetry_library? - return false unless updating_pyproject? + return false unless poetry_based? # Hit PyPi and check whether there are details for a library with a # matching name and description - details = TomlRB.parse(pyproject.content).dig("tool", "poetry") - return false unless details - index_response = Dependabot::RegistryClient.get( - url: "https://pypi.org/pypi/#{normalised_name(details['name'])}/json/" + url: "https://pypi.org/pypi/#{normalised_name(poetry_details['name'])}/json/" ) return false unless index_response.status == 200 pypi_info = JSON.parse(index_response.body)["info"] || {} - pypi_info["summary"] == details["description"] + pypi_info["summary"] == poetry_details["description"] rescue Excon::Error::Timeout false rescue URI::InvalidURIError @@ -324,6 +331,10 @@ def poetry_lock dependency_files.find { |f| f.name == "poetry.lock" } end + def poetry_details + @poetry_details ||= TomlRB.parse(pyproject.content).dig("tool", "poetry") + end + def pip_compile_files dependency_files.select { |f| f.name.end_with?(".in") } end diff --git a/python/lib/dependabot/python/update_checker/poetry_version_resolver.rb b/python/lib/dependabot/python/update_checker/poetry_version_resolver.rb index acd20b774d5..f4343fdade8 100644 --- a/python/lib/dependabot/python/update_checker/poetry_version_resolver.rb +++ b/python/lib/dependabot/python/update_checker/poetry_version_resolver.rb @@ -293,7 +293,7 @@ def set_target_dependency_req(pyproject_content, updated_requirement) pyproject_object = TomlRB.parse(pyproject_content) poetry_object = pyproject_object.dig("tool", "poetry") - Dependabot::Python::FileParser::PoetryFilesParser::POETRY_DEPENDENCY_TYPES.each do |type| + Dependabot::Python::FileParser::PyprojectFilesParser::POETRY_DEPENDENCY_TYPES.each do |type| names = poetry_object[type]&.keys || [] pkg_name = names.find { |nm| normalise(nm) == dependency.name } next unless pkg_name diff --git a/python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb b/python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb index 71e717d7b91..199fa2e2bd5 100644 --- a/python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb +++ b/python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb @@ -2,9 +2,9 @@ require "spec_helper" require "dependabot/dependency_file" -require "dependabot/python/file_parser/poetry_files_parser" +require "dependabot/python/file_parser/pyproject_files_parser" -RSpec.describe Dependabot::Python::FileParser::PoetryFilesParser do +RSpec.describe Dependabot::Python::FileParser::PyprojectFilesParser do let(:parser) { described_class.new(dependency_files: files) } let(:files) { [pyproject] } @@ -229,4 +229,30 @@ end end end + + describe "parse standard python files" do + let(:pyproject_fixture_name) { "standard_python.toml" } + + subject(:dependencies) { parser.dependency_set.dependencies } + + its(:length) { is_expected.to eq(1) } + + context "with a string declaration" do + subject(:dependency) { dependencies.first } + + it "has the right details" do + expect(dependency).to be_a(Dependabot::Dependency) + expect(dependency.name).to eq("ansys-templates") + expect(dependency.version).to eq("0.3.0") + expect(dependency.requirements).to eq( + [{ + requirement: "==0.3.0", + file: "pyproject.toml", + groups: [nil], + source: nil + }] + ) + end + end + end end diff --git a/python/spec/dependabot/python/file_updater_spec.rb b/python/spec/dependabot/python/file_updater_spec.rb index 28bfa6592af..808fe8f18ae 100644 --- a/python/spec/dependabot/python/file_updater_spec.rb +++ b/python/spec/dependabot/python/file_updater_spec.rb @@ -217,6 +217,45 @@ end end + context "with a pyproject.toml with pep621 dependencies" do + let(:dependency_files) { [pyproject] } + let(:pyproject) do + Dependabot::DependencyFile.new( + name: "pyproject.toml", + content: + fixture("pyproject_files", "standard_python.toml") + ) + end + + let(:dependency) do + Dependabot::Dependency.new( + name: "ansys-templates", + version: "0.5.0", + previous_version: "0.3.0", + package_manager: "pip", + requirements: [{ + requirement: "==0.5.0", + file: "pyproject.toml", + source: nil, + groups: ["default"] + }], + previous_requirements: [{ + requirement: "==0.3.0", + file: "pyproject.toml", + source: nil, + groups: ["default"] + }] + ) + end + + it "delegates to RequirementFileUpdater" do + expect(described_class::RequirementFileUpdater). + to receive(:new).and_call_original + expect { updated_files }.to_not(change { Dir.entries(tmp_path) }) + updated_files.each { |f| expect(f).to be_a(Dependabot::DependencyFile) } + end + end + context "with a pyproject.toml and pyproject.lock" do let(:dependency_files) { [pyproject, lockfile] } let(:pyproject) do diff --git a/python/spec/dependabot/python/update_checker_spec.rb b/python/spec/dependabot/python/update_checker_spec.rb index 37afb3e0391..8a0ad259042 100644 --- a/python/spec/dependabot/python/update_checker_spec.rb +++ b/python/spec/dependabot/python/update_checker_spec.rb @@ -49,7 +49,6 @@ content: fixture("pyproject_files", pyproject_fixture_name) ) end - let(:pyproject_fixture_name) { "poetry_exact_requirement.toml" } let(:requirements_file) do Dependabot::DependencyFile.new( name: "requirements.txt", @@ -296,17 +295,47 @@ }] end - it "delegates to PoetryVersionResolver" do - dummy_resolver = - instance_double(described_class::PoetryVersionResolver) - allow(described_class::PoetryVersionResolver).to receive(:new). - and_return(dummy_resolver) - expect(dummy_resolver). - to receive(:latest_resolvable_version). - with(requirement: ">= 2.0.0, <= 2.6.0"). - and_return(Gem::Version.new("2.5.0")) - expect(checker.latest_resolvable_version). - to eq(Gem::Version.new("2.5.0")) + let(:dependency_files) { [pyproject] } + let(:dependency_requirements) do + [{ + file: "pyproject.toml", + requirement: "2.18.0", + groups: [], + source: nil + }] + end + + context "including poetry dependencies" do + let(:pyproject_fixture_name) { "poetry_exact_requirement.toml" } + + it "delegates to PoetryVersionResolver" do + dummy_resolver = + instance_double(described_class::PoetryVersionResolver) + allow(described_class::PoetryVersionResolver).to receive(:new). + and_return(dummy_resolver) + expect(dummy_resolver). + to receive(:latest_resolvable_version). + with(requirement: ">= 2.0.0, <= 2.6.0"). + and_return(Gem::Version.new("2.5.0")) + expect(checker.latest_resolvable_version). + to eq(Gem::Version.new("2.5.0")) + end + end + + context "including pep621 dependencies" do + let(:pyproject_fixture_name) { "pep621_exact_requirement.toml" } + + it "delegates to PoetryVersionResolver" do + dummy_resolver = + instance_double(described_class::PipVersionResolver) + allow(described_class::PipVersionResolver).to receive(:new). + and_return(dummy_resolver) + expect(dummy_resolver). + to receive(:latest_resolvable_version). + and_return(Gem::Version.new("2.5.0")) + expect(checker.latest_resolvable_version). + to eq(Gem::Version.new("2.5.0")) + end end end end diff --git a/python/spec/fixtures/pyproject_files/pep621_exact_requirement.toml b/python/spec/fixtures/pyproject_files/pep621_exact_requirement.toml new file mode 100644 index 00000000000..8d380dd80e2 --- /dev/null +++ b/python/spec/fixtures/pyproject_files/pep621_exact_requirement.toml @@ -0,0 +1,13 @@ +[project] +name = "PythonProjects" +version = "2.0.0" +homepage = "https://github.com/roghu/py3_projects" +license = "MIT" +readme = "README.md" +authors = ["Dependabot "] +description = "Various small python projects." + +dependencies = [ + "python", + "requests==2.18.0" +] diff --git a/python/spec/fixtures/pyproject_files/standard_python.toml b/python/spec/fixtures/pyproject_files/standard_python.toml new file mode 100644 index 00000000000..d5fc77172bc --- /dev/null +++ b/python/spec/fixtures/pyproject_files/standard_python.toml @@ -0,0 +1,13 @@ +[build-system] +requires = ["flit_core >=3.2,<4"] +build-backend = "flit_core.buildapi" + +[project] +name = "pkgtest" +authors = [{name = "Sample", email = "sample.project@example.org"}] +license = {file = "LICENSE"} +classifiers = ["License :: OSI Approved :: MIT License"] +dynamic = ["version", "description"] +dependencies = [ + "ansys-templates==0.3.0", +] From 9fcf751b19f9d99ab809126209f5e41a6f0719a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Rodr=C3=ADguez?= Date: Mon, 5 Sep 2022 18:37:26 +0200 Subject: [PATCH 06/10] Ignore dependencies with empty requirements --- .../file_parser/pyproject_files_parser.rb | 3 ++ .../pyproject_files_parser_spec.rb | 8 +++++ .../pyproject_files/no_requirements.toml | 31 +++++++++++++++++++ 3 files changed, 42 insertions(+) create mode 100644 python/spec/fixtures/pyproject_files/no_requirements.toml diff --git a/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb index 3b77de186e6..610326d764d 100644 --- a/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb +++ b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb @@ -75,6 +75,9 @@ def pep621_dependencies # probably blocked. Ignore it. next if dep["markers"].include?("<") + # If no requirement, don't add it + next if dep["requirement"].empty? + dependencies << Dependency.new( name: normalised_name(dep["name"], dep["extras"]), diff --git a/python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb b/python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb index 199fa2e2bd5..d3edeb81ed8 100644 --- a/python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb +++ b/python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb @@ -254,5 +254,13 @@ ) end end + + context "with dependencies with empty requirements" do + let(:pyproject_fixture_name) { "no_requirements.toml" } + + subject(:dependencies) { parser.dependency_set.dependencies } + + its(:length) { is_expected.to eq(0) } + end end end diff --git a/python/spec/fixtures/pyproject_files/no_requirements.toml b/python/spec/fixtures/pyproject_files/no_requirements.toml new file mode 100644 index 00000000000..48022376300 --- /dev/null +++ b/python/spec/fixtures/pyproject_files/no_requirements.toml @@ -0,0 +1,31 @@ +[build-system] +requires = ["hatchling", "hatch-fancy-pypi-readme"] +build-backend = "hatchling.build" + +[project] +name = "httpx" +description = "The next generation HTTP client." +license = "BSD-3-Clause" +requires-python = ">=3.7" +authors = [ + { name = "Tom Christie", email = "tom@tomchristie.com" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Environment :: Web Environment", + "Framework :: AsyncIO", + "Framework :: Trio", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Topic :: Internet :: WWW/HTTP", +] +dependencies = [ + "certifi" +] From 84aa4fe0e5f416462a19e786b949221e14f78d93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Rodr=C3=ADguez?= Date: Thu, 15 Sep 2022 19:06:05 +0200 Subject: [PATCH 07/10] Don't bail out on files without dependencies field --- .../python/file_parser/pyproject_files_parser.rb | 4 ++-- .../python/file_parser/pyproject_files_parser_spec.rb | 8 ++++++++ .../spec/fixtures/pyproject_files/no_dependencies.toml | 10 ++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 python/spec/fixtures/pyproject_files/no_dependencies.toml diff --git a/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb index 610326d764d..fe91f52a63f 100644 --- a/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb +++ b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb @@ -25,7 +25,7 @@ def initialize(dependency_files:) def dependency_set dependency_set = Dependabot::FileParsers::Base::DependencySet.new - dependency_set += pyproject_dependencies + dependency_set += pyproject_dependencies if using_poetry? || using_pep621? dependency_set += lockfile_dependencies if lockfile dependency_set @@ -38,7 +38,7 @@ def dependency_set def pyproject_dependencies if using_poetry? poetry_dependencies - elsif using_pep621? + else pep621_dependencies end end diff --git a/python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb b/python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb index d3edeb81ed8..ac1e1728642 100644 --- a/python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb +++ b/python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb @@ -255,6 +255,14 @@ end end + context "without dependencies" do + let(:pyproject_fixture_name) { "no_dependencies.toml" } + + subject(:dependencies) { parser.dependency_set.dependencies } + + its(:length) { is_expected.to eq(0) } + end + context "with dependencies with empty requirements" do let(:pyproject_fixture_name) { "no_requirements.toml" } diff --git a/python/spec/fixtures/pyproject_files/no_dependencies.toml b/python/spec/fixtures/pyproject_files/no_dependencies.toml new file mode 100644 index 00000000000..c6d60e1a4a2 --- /dev/null +++ b/python/spec/fixtures/pyproject_files/no_dependencies.toml @@ -0,0 +1,10 @@ +[build-system] +requires = ["flit_core >=3.2,<4"] +build-backend = "flit_core.buildapi" + +[project] +name = "pkgtest" +authors = [{name = "Sample", email = "sample.project@example.org"}] +license = {file = "LICENSE"} +classifiers = ["License :: OSI Approved :: MIT License"] +dynamic = ["version", "description"] From 4dc71870ea8a05699d229a5fc05a143f372bf63c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Rodr=C3=ADguez?= Date: Thu, 29 Sep 2022 19:47:22 +0200 Subject: [PATCH 08/10] Fix updating libraries support Reuse the same logic used for detecting poetry libraries. Also for consistency, since standard Python does not have a caret operator, change existing poetry update checker specs to use the tilde operator instead, which is also implemented in standard python. --- .../lib/dependabot/python/update_checker.rb | 26 +++++-- .../dependabot/python/update_checker_spec.rb | 69 +++++++++++++++++-- .../standard_python_tilde_version.toml | 12 ++++ .../pyproject_files/tilde_version.toml | 11 +++ 4 files changed, 105 insertions(+), 13 deletions(-) create mode 100644 python/spec/fixtures/pyproject_files/standard_python_tilde_version.toml create mode 100644 python/spec/fixtures/pyproject_files/tilde_version.toml diff --git a/python/lib/dependabot/python/update_checker.rb b/python/lib/dependabot/python/update_checker.rb index 0b0700cd615..a82d3e8466b 100644 --- a/python/lib/dependabot/python/update_checker.rb +++ b/python/lib/dependabot/python/update_checker.rb @@ -100,8 +100,8 @@ def requirements_update_strategy # If passed in as an option (in the base class) honour that option return @requirements_update_strategy.to_sym if @requirements_update_strategy - # Otherwise, check if this is a poetry library or not - poetry_library? ? :widen_ranges : :bump_versions + # Otherwise, check if this is a library or not + library? ? :widen_ranges : :bump_versions end private @@ -268,19 +268,19 @@ def poetry_based? updating_pyproject? && !poetry_details.nil? end - def poetry_library? - return false unless poetry_based? + def library? + return unless updating_pyproject? # Hit PyPi and check whether there are details for a library with a # matching name and description index_response = Dependabot::RegistryClient.get( - url: "https://pypi.org/pypi/#{normalised_name(poetry_details['name'])}/json/" + url: "https://pypi.org/pypi/#{normalised_name(library_details['name'])}/json/" ) return false unless index_response.status == 200 pypi_info = JSON.parse(index_response.body)["info"] || {} - pypi_info["summary"] == poetry_details["description"] + pypi_info["summary"] == library_details["description"] rescue Excon::Error::Timeout false rescue URI::InvalidURIError @@ -331,8 +331,20 @@ def poetry_lock dependency_files.find { |f| f.name == "poetry.lock" } end + def library_details + @library_details ||= poetry_details || standard_details + end + def poetry_details - @poetry_details ||= TomlRB.parse(pyproject.content).dig("tool", "poetry") + @poetry_details ||= toml_content.dig("tool", "poetry") + end + + def standard_details + @standard_details ||= toml_content["project"] + end + + def toml_content + @toml_content ||= TomlRB.parse(pyproject.content) end def pip_compile_files diff --git a/python/spec/dependabot/python/update_checker_spec.rb b/python/spec/dependabot/python/update_checker_spec.rb index 8a0ad259042..da28f488713 100644 --- a/python/spec/dependabot/python/update_checker_spec.rb +++ b/python/spec/dependabot/python/update_checker_spec.rb @@ -534,9 +534,66 @@ its([:requirement]) { is_expected.to eq("==2.6.0") } end - context "when there is a pyproject.toml file" do - let(:dependency_files) { [requirements_file, pyproject] } - let(:pyproject_fixture_name) { "caret_version.toml" } + context "when there is a pyproject.toml file with poetry dependencies" do + let(:dependency_files) { [pyproject] } + let(:pyproject_fixture_name) { "tilde_version.toml" } + + context "and updating a dependency inside" do + let(:dependency) do + Dependabot::Dependency.new( + name: "requests", + version: "1.2.3", + requirements: [{ + file: "pyproject.toml", + requirement: "~1.0.0", + groups: [], + source: nil + }], + package_manager: "pip" + ) + end + + let(:pypi_url) { "https://pypi.org/simple/requests/" } + let(:pypi_response) do + fixture("pypi", "pypi_simple_response_requests.html") + end + + context "for a library" do + before do + stub_request(:get, "https://pypi.org/pypi/pendulum/json/"). + to_return( + status: 200, + body: fixture("pypi", "pypi_response_pendulum.json") + ) + end + + its([:requirement]) { is_expected.to eq(">=1.0,<2.20") } + end + + context "for a non-library" do + before do + stub_request(:get, "https://pypi.org/pypi/pendulum/json/"). + to_return(status: 404) + end + + its([:requirement]) { is_expected.to eq("~2.19.1") } + end + end + + context "and updating a dependency in an additional requirements file" do + let(:dependency_files) { super().append(requirements_file) } + + let(:dependency) { requirements_dependency } + + it "does not get affected by whether it's a library or not and updates using the :increase strategy" do + expect(subject[:requirement]).to eq("==2.6.0") + end + end + end + + context "when there is a pyproject.toml file with standard python dependencies" do + let(:dependency_files) { [pyproject] } + let(:pyproject_fixture_name) { "standard_python_tilde_version.toml" } context "and updating a dependency inside" do let(:dependency) do @@ -545,7 +602,7 @@ version: "1.2.3", requirements: [{ file: "pyproject.toml", - requirement: "^1.0.0", + requirement: "~=1.0.0", groups: [], source: nil }], @@ -567,7 +624,7 @@ ) end - its([:requirement]) { is_expected.to eq(">=1,<3") } + its([:requirement]) { is_expected.to eq(">=1.0,<2.20") } end context "for a non-library" do @@ -576,7 +633,7 @@ to_return(status: 404) end - its([:requirement]) { is_expected.to eq("^2.19.1") } + its([:requirement]) { is_expected.to eq("~=2.19.1") } end end diff --git a/python/spec/fixtures/pyproject_files/standard_python_tilde_version.toml b/python/spec/fixtures/pyproject_files/standard_python_tilde_version.toml new file mode 100644 index 00000000000..949b1d5c713 --- /dev/null +++ b/python/spec/fixtures/pyproject_files/standard_python_tilde_version.toml @@ -0,0 +1,12 @@ +[project] +name = "pendulum" +version = "2.0.0" +homepage = "https://github.com/roghu/py3_projects" +license = "MIT" +readme = "README.md" +authors = ["Dependabot "] +description = "Python datetimes made easy" + +dependencies = [ + "requests~=1.0.0" +] diff --git a/python/spec/fixtures/pyproject_files/tilde_version.toml b/python/spec/fixtures/pyproject_files/tilde_version.toml new file mode 100644 index 00000000000..446f97913ea --- /dev/null +++ b/python/spec/fixtures/pyproject_files/tilde_version.toml @@ -0,0 +1,11 @@ +[tool.poetry] +name = "pendulum" +version = "2.0.0" +homepage = "https://github.com/roghu/py3_projects" +license = "MIT" +readme = "README.md" +authors = ["Dependabot "] +description = "Python datetimes made easy" + +[tool.poetry.dependencies] +requests = "~1.0.0" From a32e88a7cdb974082a6c1cb0bd36df4b44e90c62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Rodr=C3=ADguez?= Date: Tue, 11 Oct 2022 19:17:42 +0200 Subject: [PATCH 09/10] Avoid dealing with PDM projects for now --- python/lib/dependabot/python/file_fetcher.rb | 6 +- .../file_parser/pyproject_files_parser.rb | 15 ++ .../dependabot/python/file_fetcher_spec.rb | 30 ++++ .../pyproject_files_parser_spec.rb | 19 +++ .../github/contents_python_pdm_lock.json | 18 +++ ...ontents_python_pyproject_and_pdm_lock.json | 146 ++++++++++++++++++ .../fixtures/pyproject_files/pdm_example.toml | 31 ++++ .../fixtures/pyproject_locks/pdm_example.lock | 140 +++++++++++++++++ 8 files changed, 404 insertions(+), 1 deletion(-) create mode 100644 python/spec/fixtures/github/contents_python_pdm_lock.json create mode 100644 python/spec/fixtures/github/contents_python_pyproject_and_pdm_lock.json create mode 100644 python/spec/fixtures/pyproject_files/pdm_example.toml create mode 100644 python/spec/fixtures/pyproject_locks/pdm_example.lock diff --git a/python/lib/dependabot/python/file_fetcher.rb b/python/lib/dependabot/python/file_fetcher.rb index 1318c875d5a..1b3ca181795 100644 --- a/python/lib/dependabot/python/file_fetcher.rb +++ b/python/lib/dependabot/python/file_fetcher.rb @@ -69,7 +69,7 @@ def pipenv_files end def pyproject_files - [pyproject, pyproject_lock, poetry_lock].compact + [pyproject, pyproject_lock, poetry_lock, pdm_lock].compact end def requirement_files @@ -141,6 +141,10 @@ def poetry_lock @poetry_lock ||= fetch_file_if_present("poetry.lock") end + def pdm_lock + @pdm_lock ||= fetch_file_if_present("pdm.lock") + end + def requirements_txt_files req_txt_and_in_files.select { |f| f.name.end_with?(".txt") } end diff --git a/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb index fe91f52a63f..54da79f9cf4 100644 --- a/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb +++ b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb @@ -70,6 +70,12 @@ def poetry_dependencies def pep621_dependencies dependencies = Dependabot::FileParsers::Base::DependencySet.new + # PDM is not yet supported, so we want to ignore it for now because in + # the current state of things, going on would result in updating + # pyproject.toml but leaving pdm.lock out of sync, which is + # undesirable. Leave PDM alone until properly supported + return dependencies if using_pdm? + parsed_pep621_dependencies.each do |dep| # If a requirement has a `<` or `<=` marker then updating it is # probably blocked. Ignore it. @@ -123,6 +129,10 @@ def using_pep621? !parsed_pyproject.dig("project", "dependencies").nil? end + def using_pdm? + using_pep621? && pdm_lock + end + # Create a DependencySet where each element has no requirement. Any # requirements will be added when combining the DependencySet with # other DependencySets. @@ -226,6 +236,11 @@ def poetry_lock @poetry_lock ||= dependency_files.find { |f| f.name == "poetry.lock" } end + + def pdm_lock + @pdm_lock ||= + dependency_files.find { |f| f.name == "pdm.lock" } + end end end end diff --git a/python/spec/dependabot/python/file_fetcher_spec.rb b/python/spec/dependabot/python/file_fetcher_spec.rb index 57160140439..1570aabd413 100644 --- a/python/spec/dependabot/python/file_fetcher_spec.rb +++ b/python/spec/dependabot/python/file_fetcher_spec.rb @@ -372,6 +372,36 @@ end end + context "with a pyproject.toml and pdm.lock files" do + let(:repo_contents) do + fixture("github", "contents_python_pyproject_and_pdm_lock.json") + end + + before do + stub_request(:get, url + "pyproject.toml?ref=sha"). + with(headers: { "Authorization" => "token token" }). + to_return( + status: 200, + body: fixture("github", "contents_python_pyproject.json"), + headers: { "content-type" => "application/json" } + ) + + stub_request(:get, url + "pdm.lock?ref=sha"). + with(headers: { "Authorization" => "token token" }). + to_return( + status: 200, + body: fixture("github", "contents_python_pdm_lock.json"), + headers: { "content-type" => "application/json" } + ) + end + + it "fetches the pyproject.toml and pdm.lock files" do + expect(file_fetcher_instance.files.count).to eq(2) + expect(file_fetcher_instance.files.map(&:name)). + to match_array(%w(pyproject.toml pdm.lock)) + end + end + context "with no setup.py, requirements.txt or Pipfile" do let(:repo_contents) { "[]" } diff --git a/python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb b/python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb index ac1e1728642..28cb0ba78c2 100644 --- a/python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb +++ b/python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb @@ -270,5 +270,24 @@ its(:length) { is_expected.to eq(0) } end + + context "with a PDM project" do + let(:pyproject_fixture_name) { "pdm_example.toml" } + let(:pdm_lock) do + Dependabot::DependencyFile.new( + name: "pdm.lock", + content: pdm_lock_body + ) + end + let(:pdm_lock_body) do + fixture("pyproject_locks", pyproject_lock_fixture_name) + end + let(:pyproject_lock_fixture_name) { "pdm_example.lock" } + let(:files) { [pyproject, pdm_lock] } + + subject(:dependencies) { parser.dependency_set.dependencies } + + its(:length) { is_expected.to eq(0) } + end end end diff --git a/python/spec/fixtures/github/contents_python_pdm_lock.json b/python/spec/fixtures/github/contents_python_pdm_lock.json new file mode 100644 index 00000000000..4b0581a21f2 --- /dev/null +++ b/python/spec/fixtures/github/contents_python_pdm_lock.json @@ -0,0 +1,18 @@ +{ + "name": "pdm.lock", + "path": "pdm.lock", + "sha": "31a43a994d33ec6fab0a26a08b7313c6836e0822", + "size": 3281, + "url": "https://api.github.com/repos/pdm-project/setup-pdm/contents/pdm.lock?ref=main", + "html_url": "https://github.com/pdm-project/setup-pdm/blob/main/pdm.lock", + "git_url": "https://api.github.com/repos/pdm-project/setup-pdm/git/blobs/31a43a994d33ec6fab0a26a08b7313c6836e0822", + "download_url": "https://raw.githubusercontent.com/pdm-project/setup-pdm/main/pdm.lock", + "type": "file", + "content": "W1twYWNrYWdlXV0KbmFtZSA9ICJjZXJ0aWZpIgp2ZXJzaW9uID0gIjIwMjIu\nNi4xNSIKcmVxdWlyZXNfcHl0aG9uID0gIj49My42IgpzdW1tYXJ5ID0gIlB5\ndGhvbiBwYWNrYWdlIGZvciBwcm92aWRpbmcgTW96aWxsYSdzIENBIEJ1bmRs\nZS4iCgpbW3BhY2thZ2VdXQpuYW1lID0gImNoYXJkZXQiCnZlcnNpb24gPSAi\nNS4wLjAiCnJlcXVpcmVzX3B5dGhvbiA9ICI+PTMuNiIKc3VtbWFyeSA9ICJV\nbml2ZXJzYWwgZW5jb2RpbmcgZGV0ZWN0b3IgZm9yIFB5dGhvbiAzIgoKW1tw\nYWNrYWdlXV0KbmFtZSA9ICJpZG5hIgp2ZXJzaW9uID0gIjMuMyIKcmVxdWly\nZXNfcHl0aG9uID0gIj49My41IgpzdW1tYXJ5ID0gIkludGVybmF0aW9uYWxp\nemVkIERvbWFpbiBOYW1lcyBpbiBBcHBsaWNhdGlvbnMgKElETkEpIgoKW1tw\nYWNrYWdlXV0KbmFtZSA9ICJweXR6Igp2ZXJzaW9uID0gIjIwMjIuMSIKc3Vt\nbWFyeSA9ICJXb3JsZCB0aW1lem9uZSBkZWZpbml0aW9ucywgbW9kZXJuIGFu\nZCBoaXN0b3JpY2FsIgoKW1twYWNrYWdlXV0KbmFtZSA9ICJ1cmxsaWIzIgp2\nZXJzaW9uID0gIjEuMjYuMTAiCnJlcXVpcmVzX3B5dGhvbiA9ICI+PTIuNywg\nIT0zLjAuKiwgIT0zLjEuKiwgIT0zLjIuKiwgIT0zLjMuKiwgIT0zLjQuKiwg\nIT0zLjUuKiwgPDQiCnN1bW1hcnkgPSAiSFRUUCBsaWJyYXJ5IHdpdGggdGhy\nZWFkLXNhZmUgY29ubmVjdGlvbiBwb29saW5nLCBmaWxlIHBvc3QsIGFuZCBt\nb3JlLiIKClttZXRhZGF0YV0KbG9ja192ZXJzaW9uID0gIjQuMCIKY29udGVu\ndF9oYXNoID0gInNoYTI1Njo5Mzg2YjNkNmFjOTU5NTA4ZWYzY2Y3NWZkYWI3\nYzgyYmQ0NDBjMGE3OTg0MWEwNzg2ZDcxYzFlNDJhNThiMjRkIgoKW21ldGFk\nYXRhLmZpbGVzXQoiY2VydGlmaSAyMDIyLjYuMTUiID0gWwogICAge3VybCA9\nICJodHRwczovL2ZpbGVzLnB5dGhvbmhvc3RlZC5vcmcvcGFja2FnZXMvZTkv\nMDYvZDNkMzY3YjdhZjYzMDViMTZmMGQyOGFlMmFhZWI4NjE1NGZhOTFmMTQ0\nZjAzNmMyZDUwMDJhNWEyMDJiL2NlcnRpZmktMjAyMi42LjE1LXB5My1ub25l\nLWFueS53aGwiLCBoYXNoID0gInNoYTI1NjpmZTg2NDE1ZDU1ZTg0NzE5ZDc1\nZjhiNjk0MTRmNjQzOGFjMzU0N2QyMDc4YWI5MWI2N2U3NzllZjY5Mzc4NDEy\nIn0sCiAgICB7dXJsID0gImh0dHBzOi8vZmlsZXMucHl0aG9uaG9zdGVkLm9y\nZy9wYWNrYWdlcy9jYy84NS8zMTlhOGE2ODRlOGFjNmQ4N2ExMTkzMDkwZTA2\nYjZiYmIzMDI3MTc0OTYzODBlMjI1ZWUxMDQ4N2M4ODgvY2VydGlmaS0yMDIy\nLjYuMTUudGFyLmd6IiwgaGFzaCA9ICJzaGEyNTY6ODRjODVhOTA3OGIxMTEw\nNWYwNGYzMDM2YTk0ODJhZTEwZTQ2MjE2MTZkYjMxM2ZlMDQ1ZGQyNDc0M2Ew\nODIwZCJ9LApdCiJjaGFyZGV0IDUuMC4wIiA9IFsKICAgIHt1cmwgPSAiaHR0\ncHM6Ly9maWxlcy5weXRob25ob3N0ZWQub3JnL3BhY2thZ2VzLzRjL2QxLzFi\nOTZkZDY5ZmE0MmYyMGI3MDcwMWI1Y2Q0MmE3NWRkNWYwYzdhMjRkYzBhYmZl\nZjM1Y2MxNDYyMTBkYy9jaGFyZGV0LTUuMC4wLXB5My1ub25lLWFueS53aGwi\nLCBoYXNoID0gInNoYTI1NjpkM2U2NGYwMjJkMjU0MTgzMDAxZWNjYzVkYjQw\nNDA1MjBjMGYyM2IxYTNmMzNkNjQxM2UwOTllYjdmMTI2NTU3In0sCiAgICB7\ndXJsID0gImh0dHBzOi8vZmlsZXMucHl0aG9uaG9zdGVkLm9yZy9wYWNrYWdl\ncy8zMS9hMi8xMmMwOTA3MTNiM2QwZTE0MWYzNjcyMzZkM2E4YmRjM2U1ZmNh\nMGQ4M2ZmMzY0N2FmNDg5MmMxNmMyMDUvY2hhcmRldC01LjAuMC50YXIuZ3oi\nLCBoYXNoID0gInNoYTI1NjowMzY4ZGYyYmZkNzhiNWZjMjA1NzJiYjRlOWJi\nN2ZiNTNlMmMwOTRmNjBhZTk5OTMzMzllODY3MWQwYWZiOGFhIn0sCl0KImlk\nbmEgMy4zIiA9IFsKICAgIHt1cmwgPSAiaHR0cHM6Ly9maWxlcy5weXRob25o\nb3N0ZWQub3JnL3BhY2thZ2VzLzA0L2EyL2Q5MThkY2QyMjM1NGQ4OTU4ZmUx\nMTNlMWEzNjMwMTM3ZTBmYzhiNDQ4NTlhZGUzMDYzOTgyZWFjZDJhNC9pZG5h\nLTMuMy1weTMtbm9uZS1hbnkud2hsIiwgaGFzaCA9ICJzaGEyNTY6ODRkOWRk\nMDQ3ZmZhODA1OTZlMGYyNDZlMmVhYjBiMzkxNzg4YjA1MDM1ODRlODk0NWYy\nMzY4MjU2ZDI3MzVmZiJ9LAogICAge3VybCA9ICJodHRwczovL2ZpbGVzLnB5\ndGhvbmhvc3RlZC5vcmcvcGFja2FnZXMvNjIvMDgvZTNmYzdjODE2MTA5MGY3\nNDJmNTA0ZjQwYjFiY2NiZmM1NDRkNGE0ZTA5ZWI3NzRiZjQwYWFmY2U1NDM2\nL2lkbmEtMy4zLnRhci5neiIsIGhhc2ggPSAic2hhMjU2OjlkNjQzZmYwYTU1\nYjc2MmQ1Y2RiMTI0YjhlYWE5OWM2NjMyMmUyMTU3YjY5MTYwYmMzMjc5NmU4\nMjQzNjBlNmQifSwKXQoicHl0eiAyMDIyLjEiID0gWwogICAge3VybCA9ICJo\ndHRwczovL2ZpbGVzLnB5dGhvbmhvc3RlZC5vcmcvcGFja2FnZXMvNjAvMmUv\nZGVjMWNjMThjNTFiOGRmMzNjN2M0ZDBhMzIxYjA4NGNmMzhlMTczM2I5OGY5\nZDE1MDE4ODgwZmI0OTcwL3B5dHotMjAyMi4xLXB5Mi5weTMtbm9uZS1hbnku\nd2hsIiwgaGFzaCA9ICJzaGEyNTY6ZTY4OTg1OTg1Mjk2ZDlhNjZhODgxZWIz\nMTkzYjA5MDYyNDYyNDUyOTRhODgxZTdjOGFmZTYyMzg2NmFjNmE1YyJ9LAog\nICAge3VybCA9ICJodHRwczovL2ZpbGVzLnB5dGhvbmhvc3RlZC5vcmcvcGFj\na2FnZXMvMmYvNWYvYTBmNjUzMzExYWRmZjkwNWJiY2FhNmQzZGZhZjk3ZWRj\nZjRkMjYxMzgzOTNjNmNjZDM3YTQ4NDg1MWZiL3B5dHotMjAyMi4xLnRhci5n\neiIsIGhhc2ggPSAic2hhMjU2OjFlNzYwZTJmZTZhODE2M2JjMGIzZDlhMTlj\nNGY4NDM0MmFmYTBhMmFmZmViZmFhODRiMDFiOTc4YTAyZWNhYTcifSwKXQoi\ndXJsbGliMyAxLjI2LjEwIiA9IFsKICAgIHt1cmwgPSAiaHR0cHM6Ly9maWxl\ncy5weXRob25ob3N0ZWQub3JnL3BhY2thZ2VzLzY4LzQ3LzkzZDNkMjhlOTdj\nNzU3N2Y1NjM5MDM5MDc5MTJmNGIzODA0MDU0ZTQ4NzdhNWJhNjY1MWY3MTgy\nYzUzYi91cmxsaWIzLTEuMjYuMTAtcHkyLnB5My1ub25lLWFueS53aGwiLCBo\nYXNoID0gInNoYTI1Njo4Mjk4ZDZkNTZkMzliZTBlM2JjMTNjMWM5N2QxMzNm\nOWI0NWQ3OTcxNjlhMGUxMWNkZDBlMDQ4OWQ3ODZmN2VjIn0sCiAgICB7dXJs\nID0gImh0dHBzOi8vZmlsZXMucHl0aG9uaG9zdGVkLm9yZy9wYWNrYWdlcy8y\nNS8zNi9mMDU2ZTVmMTM4OTAwNGNmODg2YmI3YTg1MTQwNzdmMjQyMjQyMzhh\nNzUzNDQ5N2MwMTRhNmI5YWM3NzAvdXJsbGliMy0xLjI2LjEwLnRhci5neiIs\nIGhhc2ggPSAic2hhMjU2Ojg3OWJhNGQxZTg5NjU0ZDk3NjljZTEzMTIxZTBm\nOTQzMTBlYTMyZThkMmY4Y2Y1ODdiNzdjMDhiYmNkYjMwZDYifSwKXQo=\n", + "encoding": "base64", + "_links": { + "self": "https://api.github.com/repos/pdm-project/setup-pdm/contents/pdm.lock?ref=main", + "git": "https://api.github.com/repos/pdm-project/setup-pdm/git/blobs/31a43a994d33ec6fab0a26a08b7313c6836e0822", + "html": "https://github.com/pdm-project/setup-pdm/blob/main/pdm.lock" + } +} diff --git a/python/spec/fixtures/github/contents_python_pyproject_and_pdm_lock.json b/python/spec/fixtures/github/contents_python_pyproject_and_pdm_lock.json new file mode 100644 index 00000000000..3b3eb305627 --- /dev/null +++ b/python/spec/fixtures/github/contents_python_pyproject_and_pdm_lock.json @@ -0,0 +1,146 @@ +[ + { + "name": ".gitignore", + "path": ".gitignore", + "sha": "69cb8deeb32e391d4c7468bcb946d1dcc4a18f7e", + "size": 50, + "url": "https://api.github.com/repos/pdm-project/setup-pdm/contents/.gitignore?ref=main", + "html_url": "https://github.com/pdm-project/setup-pdm/blob/main/.gitignore", + "git_url": "https://api.github.com/repos/pdm-project/setup-pdm/git/blobs/69cb8deeb32e391d4c7468bcb946d1dcc4a18f7e", + "download_url": "https://raw.githubusercontent.com/pdm-project/setup-pdm/main/.gitignore", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/pdm-project/setup-pdm/contents/.gitignore?ref=main", + "git": "https://api.github.com/repos/pdm-project/setup-pdm/git/blobs/69cb8deeb32e391d4c7468bcb946d1dcc4a18f7e", + "html": "https://github.com/pdm-project/setup-pdm/blob/main/.gitignore" + } + }, + { + "name": "LICENSE", + "path": "LICENSE", + "sha": "84fe533754a50b4ebba98d31711e44fda16e78ec", + "size": 1068, + "url": "https://api.github.com/repos/pdm-project/setup-pdm/contents/LICENSE?ref=main", + "html_url": "https://github.com/pdm-project/setup-pdm/blob/main/LICENSE", + "git_url": "https://api.github.com/repos/pdm-project/setup-pdm/git/blobs/84fe533754a50b4ebba98d31711e44fda16e78ec", + "download_url": "https://raw.githubusercontent.com/pdm-project/setup-pdm/main/LICENSE", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/pdm-project/setup-pdm/contents/LICENSE?ref=main", + "git": "https://api.github.com/repos/pdm-project/setup-pdm/git/blobs/84fe533754a50b4ebba98d31711e44fda16e78ec", + "html": "https://github.com/pdm-project/setup-pdm/blob/main/LICENSE" + } + }, + { + "name": "README.md", + "path": "README.md", + "sha": "0aef9494ec8522bd282da9d9fdfabce2e37772e9", + "size": 1489, + "url": "https://api.github.com/repos/pdm-project/setup-pdm/contents/README.md?ref=main", + "html_url": "https://github.com/pdm-project/setup-pdm/blob/main/README.md", + "git_url": "https://api.github.com/repos/pdm-project/setup-pdm/git/blobs/0aef9494ec8522bd282da9d9fdfabce2e37772e9", + "download_url": "https://raw.githubusercontent.com/pdm-project/setup-pdm/main/README.md", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/pdm-project/setup-pdm/contents/README.md?ref=main", + "git": "https://api.github.com/repos/pdm-project/setup-pdm/git/blobs/0aef9494ec8522bd282da9d9fdfabce2e37772e9", + "html": "https://github.com/pdm-project/setup-pdm/blob/main/README.md" + } + }, + { + "name": "action.yml", + "path": "action.yml", + "sha": "ce06edd0371247671900428088c35db33b47ea40", + "size": 1119, + "url": "https://api.github.com/repos/pdm-project/setup-pdm/contents/action.yml?ref=main", + "html_url": "https://github.com/pdm-project/setup-pdm/blob/main/action.yml", + "git_url": "https://api.github.com/repos/pdm-project/setup-pdm/git/blobs/ce06edd0371247671900428088c35db33b47ea40", + "download_url": "https://raw.githubusercontent.com/pdm-project/setup-pdm/main/action.yml", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/pdm-project/setup-pdm/contents/action.yml?ref=main", + "git": "https://api.github.com/repos/pdm-project/setup-pdm/git/blobs/ce06edd0371247671900428088c35db33b47ea40", + "html": "https://github.com/pdm-project/setup-pdm/blob/main/action.yml" + } + }, + { + "name": "package.json", + "path": "package.json", + "sha": "bdbc5fb8c2f171476b749c74b3c18886402fc58f", + "size": 668, + "url": "https://api.github.com/repos/pdm-project/setup-pdm/contents/package.json?ref=main", + "html_url": "https://github.com/pdm-project/setup-pdm/blob/main/package.json", + "git_url": "https://api.github.com/repos/pdm-project/setup-pdm/git/blobs/bdbc5fb8c2f171476b749c74b3c18886402fc58f", + "download_url": "https://raw.githubusercontent.com/pdm-project/setup-pdm/main/package.json", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/pdm-project/setup-pdm/contents/package.json?ref=main", + "git": "https://api.github.com/repos/pdm-project/setup-pdm/git/blobs/bdbc5fb8c2f171476b749c74b3c18886402fc58f", + "html": "https://github.com/pdm-project/setup-pdm/blob/main/package.json" + } + }, + { + "name": "pdm.lock", + "path": "pdm.lock", + "sha": "31a43a994d33ec6fab0a26a08b7313c6836e0822", + "size": 3281, + "url": "https://api.github.com/repos/pdm-project/setup-pdm/contents/pdm.lock?ref=main", + "html_url": "https://github.com/pdm-project/setup-pdm/blob/main/pdm.lock", + "git_url": "https://api.github.com/repos/pdm-project/setup-pdm/git/blobs/31a43a994d33ec6fab0a26a08b7313c6836e0822", + "download_url": "https://raw.githubusercontent.com/pdm-project/setup-pdm/main/pdm.lock", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/pdm-project/setup-pdm/contents/pdm.lock?ref=main", + "git": "https://api.github.com/repos/pdm-project/setup-pdm/git/blobs/31a43a994d33ec6fab0a26a08b7313c6836e0822", + "html": "https://github.com/pdm-project/setup-pdm/blob/main/pdm.lock" + } + }, + { + "name": "pyproject.toml", + "path": "pyproject.toml", + "sha": "ef878626903d08844f17dec880ec944c2312ccd5", + "size": 526, + "url": "https://api.github.com/repos/pdm-project/setup-pdm/contents/pyproject.toml?ref=main", + "html_url": "https://github.com/pdm-project/setup-pdm/blob/main/pyproject.toml", + "git_url": "https://api.github.com/repos/pdm-project/setup-pdm/git/blobs/ef878626903d08844f17dec880ec944c2312ccd5", + "download_url": "https://raw.githubusercontent.com/pdm-project/setup-pdm/main/pyproject.toml", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/pdm-project/setup-pdm/contents/pyproject.toml?ref=main", + "git": "https://api.github.com/repos/pdm-project/setup-pdm/git/blobs/ef878626903d08844f17dec880ec944c2312ccd5", + "html": "https://github.com/pdm-project/setup-pdm/blob/main/pyproject.toml" + } + }, + { + "name": "test.py", + "path": "test.py", + "sha": "f4f28c78d3f907d97dc3d74d411e9d28aa8df4af", + "size": 1052, + "url": "https://api.github.com/repos/pdm-project/setup-pdm/contents/test.py?ref=main", + "html_url": "https://github.com/pdm-project/setup-pdm/blob/main/test.py", + "git_url": "https://api.github.com/repos/pdm-project/setup-pdm/git/blobs/f4f28c78d3f907d97dc3d74d411e9d28aa8df4af", + "download_url": "https://raw.githubusercontent.com/pdm-project/setup-pdm/main/test.py", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/pdm-project/setup-pdm/contents/test.py?ref=main", + "git": "https://api.github.com/repos/pdm-project/setup-pdm/git/blobs/f4f28c78d3f907d97dc3d74d411e9d28aa8df4af", + "html": "https://github.com/pdm-project/setup-pdm/blob/main/test.py" + } + }, + { + "name": "tsconfig.json", + "path": "tsconfig.json", + "sha": "9e1ee252d05cfcafa70d459f87edc45cb2262926", + "size": 5523, + "url": "https://api.github.com/repos/pdm-project/setup-pdm/contents/tsconfig.json?ref=main", + "html_url": "https://github.com/pdm-project/setup-pdm/blob/main/tsconfig.json", + "git_url": "https://api.github.com/repos/pdm-project/setup-pdm/git/blobs/9e1ee252d05cfcafa70d459f87edc45cb2262926", + "download_url": "https://raw.githubusercontent.com/pdm-project/setup-pdm/main/tsconfig.json", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/pdm-project/setup-pdm/contents/tsconfig.json?ref=main", + "git": "https://api.github.com/repos/pdm-project/setup-pdm/git/blobs/9e1ee252d05cfcafa70d459f87edc45cb2262926", + "html": "https://github.com/pdm-project/setup-pdm/blob/main/tsconfig.json" + } + } +] diff --git a/python/spec/fixtures/pyproject_files/pdm_example.toml b/python/spec/fixtures/pyproject_files/pdm_example.toml new file mode 100644 index 00000000000..b1846a4ed57 --- /dev/null +++ b/python/spec/fixtures/pyproject_files/pdm_example.toml @@ -0,0 +1,31 @@ +[project] +name = "demo" +version = "0.1.0" +description = "Fixture project from https://github.com/pdm-project/tox-pdm" +authors = [ + {name = "Frost Ming", email = "mianghong@gmail.com"}, +] +dependencies = [ + "requests~=2.25", +] +requires-python = ">=3.6" +dynamic = ["classifiers"] +license = {text = "MIT"} + +[project.urls] +homepage = "" + + +[project.optional-dependencies] +lint = [ + "flake8~=3.8", +] +[build-system] +requires = ["pdm-pep517"] +build-backend = "pdm.pep517.api" + +[tool] +[tool.pdm] +[tool.pdm.scripts] +lint = "flake8 demo.py" +lint-shell = {shell = "flake8 demo.py"} diff --git a/python/spec/fixtures/pyproject_locks/pdm_example.lock b/python/spec/fixtures/pyproject_locks/pdm_example.lock new file mode 100644 index 00000000000..c4faae860d9 --- /dev/null +++ b/python/spec/fixtures/pyproject_locks/pdm_example.lock @@ -0,0 +1,140 @@ +[[package]] +name = "certifi" +version = "2022.9.24" +requires_python = ">=3.6" +summary = "Python package for providing Mozilla's CA Bundle." + +[[package]] +name = "charset-normalizer" +version = "2.0.12" +requires_python = ">=3.5.0" +summary = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." + +[[package]] +name = "flake8" +version = "3.9.2" +requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +summary = "the modular source code checker: pep8 pyflakes and co" +dependencies = [ + "importlib-metadata; python_version < \"3.8\"", + "mccabe<0.7.0,>=0.6.0", + "pycodestyle<2.8.0,>=2.7.0", + "pyflakes<2.4.0,>=2.3.0", +] + +[[package]] +name = "idna" +version = "3.4" +requires_python = ">=3.5" +summary = "Internationalized Domain Names in Applications (IDNA)" + +[[package]] +name = "importlib-metadata" +version = "4.8.3" +requires_python = ">=3.6" +summary = "Read metadata from Python packages" +dependencies = [ + "typing-extensions>=3.6.4; python_version < \"3.8\"", + "zipp>=0.5", +] + +[[package]] +name = "mccabe" +version = "0.6.1" +summary = "McCabe checker, plugin for flake8" + +[[package]] +name = "pycodestyle" +version = "2.7.0" +requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +summary = "Python style guide checker" + +[[package]] +name = "pyflakes" +version = "2.3.1" +requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +summary = "passive checker of Python programs" + +[[package]] +name = "requests" +version = "2.27.1" +requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +summary = "Python HTTP for Humans." +dependencies = [ + "certifi>=2017.4.17", + "charset-normalizer~=2.0.0; python_version >= \"3\"", + "idna<4,>=2.5; python_version >= \"3\"", + "urllib3<1.27,>=1.21.1", +] + +[[package]] +name = "typing-extensions" +version = "4.1.1" +requires_python = ">=3.6" +summary = "Backported and Experimental Type Hints for Python 3.6+" + +[[package]] +name = "urllib3" +version = "1.26.12" +requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4" +summary = "HTTP library with thread-safe connection pooling, file post, and more." + +[[package]] +name = "zipp" +version = "3.6.0" +requires_python = ">=3.6" +summary = "Backport of pathlib-compatible object wrapper for zip files" + +[metadata] +lock_version = "4.0" +content_hash = "sha256:14105ab2be152f8aab79a29878099026d06375acc7b647029d24ef620399442b" + +[metadata.files] +"certifi 2022.9.24" = [ + {url = "https://files.pythonhosted.org/packages/1d/38/fa96a426e0c0e68aabc68e896584b83ad1eec779265a028e156ce509630e/certifi-2022.9.24-py3-none-any.whl", hash = "sha256:90c1a32f1d68f940488354e36370f6cca89f0f106db09518524c88d6ed83f382"}, + {url = "https://files.pythonhosted.org/packages/cb/a4/7de7cd59e429bd0ee6521ba58a75adaec136d32f91a761b28a11d8088d44/certifi-2022.9.24.tar.gz", hash = "sha256:0d9c601124e5a6ba9712dbc60d9c53c21e34f5f641fe83002317394311bdce14"}, +] +"charset-normalizer 2.0.12" = [ + {url = "https://files.pythonhosted.org/packages/06/b3/24afc8868eba069a7f03650ac750a778862dc34941a4bebeb58706715726/charset_normalizer-2.0.12-py3-none-any.whl", hash = "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df"}, + {url = "https://files.pythonhosted.org/packages/56/31/7bcaf657fafb3c6db8c787a865434290b726653c912085fbd371e9b92e1c/charset-normalizer-2.0.12.tar.gz", hash = "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597"}, +] +"flake8 3.9.2" = [ + {url = "https://files.pythonhosted.org/packages/9e/47/15b267dfe7e03dca4c4c06e7eadbd55ef4dfd368b13a0bab36d708b14366/flake8-3.9.2.tar.gz", hash = "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b"}, + {url = "https://files.pythonhosted.org/packages/fc/80/35a0716e5d5101e643404dabd20f07f5528a21f3ef4032d31a49c913237b/flake8-3.9.2-py2.py3-none-any.whl", hash = "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907"}, +] +"idna 3.4" = [ + {url = "https://files.pythonhosted.org/packages/8b/e1/43beb3d38dba6cb420cefa297822eac205a277ab43e5ba5d5c46faf96438/idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, + {url = "https://files.pythonhosted.org/packages/fc/34/3030de6f1370931b9dbb4dad48f6ab1015ab1d32447850b9fc94e60097be/idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, +] +"importlib-metadata 4.8.3" = [ + {url = "https://files.pythonhosted.org/packages/85/ed/e65128cc5cb1580f22ee3009d9187ecdfcc43ffb3b581fe854b24e87d8e7/importlib_metadata-4.8.3.tar.gz", hash = "sha256:766abffff765960fcc18003801f7044eb6755ffae4521c8e8ce8e83b9c9b0668"}, + {url = "https://files.pythonhosted.org/packages/a0/a1/b153a0a4caf7a7e3f15c2cd56c7702e2cf3d89b1b359d1f1c5e59d68f4ce/importlib_metadata-4.8.3-py3-none-any.whl", hash = "sha256:65a9576a5b2d58ca44d133c42a241905cc45e34d2c06fd5ba2bafa221e5d7b5e"}, +] +"mccabe 0.6.1" = [ + {url = "https://files.pythonhosted.org/packages/06/18/fa675aa501e11d6d6ca0ae73a101b2f3571a565e0f7d38e062eec18a91ee/mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, + {url = "https://files.pythonhosted.org/packages/87/89/479dc97e18549e21354893e4ee4ef36db1d237534982482c3681ee6e7b57/mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, +] +"pycodestyle 2.7.0" = [ + {url = "https://files.pythonhosted.org/packages/02/b3/c832123f2699892c715fcdfebb1a8fdeffa11bb7b2350e46ecdd76b45a20/pycodestyle-2.7.0.tar.gz", hash = "sha256:c389c1d06bf7904078ca03399a4816f974a1d590090fecea0c63ec26ebaf1cef"}, + {url = "https://files.pythonhosted.org/packages/de/cc/227251b1471f129bc35e966bb0fceb005969023926d744139642d847b7ae/pycodestyle-2.7.0-py2.py3-none-any.whl", hash = "sha256:514f76d918fcc0b55c6680472f0a37970994e07bbb80725808c17089be302068"}, +] +"pyflakes 2.3.1" = [ + {url = "https://files.pythonhosted.org/packages/6c/11/2a745612f1d3cbbd9c69ba14b1b43a35a2f5c3c81cd0124508c52c64307f/pyflakes-2.3.1-py2.py3-none-any.whl", hash = "sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3"}, + {url = "https://files.pythonhosted.org/packages/a8/0f/0dc480da9162749bf629dca76570972dd9cce5bedc60196a3c912875c87d/pyflakes-2.3.1.tar.gz", hash = "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db"}, +] +"requests 2.27.1" = [ + {url = "https://files.pythonhosted.org/packages/2d/61/08076519c80041bc0ffa1a8af0cbd3bf3e2b62af10435d269a9d0f40564d/requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"}, + {url = "https://files.pythonhosted.org/packages/60/f3/26ff3767f099b73e0efa138a9998da67890793bfa475d8278f84a30fec77/requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"}, +] +"typing-extensions 4.1.1" = [ + {url = "https://files.pythonhosted.org/packages/45/6b/44f7f8f1e110027cf88956b59f2fad776cca7e1704396d043f89effd3a0e/typing_extensions-4.1.1-py3-none-any.whl", hash = "sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2"}, + {url = "https://files.pythonhosted.org/packages/b1/5a/8b5fbb891ef3f81fc923bf3cb4a578c0abf9471eb50ce0f51c74212182ab/typing_extensions-4.1.1.tar.gz", hash = "sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42"}, +] +"urllib3 1.26.12" = [ + {url = "https://files.pythonhosted.org/packages/6f/de/5be2e3eed8426f871b170663333a0f627fc2924cc386cd41be065e7ea870/urllib3-1.26.12-py2.py3-none-any.whl", hash = "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997"}, + {url = "https://files.pythonhosted.org/packages/b2/56/d87d6d3c4121c0bcec116919350ca05dc3afd2eeb7dc88d07e8083f8ea94/urllib3-1.26.12.tar.gz", hash = "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e"}, +] +"zipp 3.6.0" = [ + {url = "https://files.pythonhosted.org/packages/02/bf/0d03dbdedb83afec081fefe86cae3a2447250ef1a81ac601a9a56e785401/zipp-3.6.0.tar.gz", hash = "sha256:71c644c5369f4a6e07636f0aa966270449561fcea2e3d6747b8d23efaa9d7832"}, + {url = "https://files.pythonhosted.org/packages/bd/df/d4a4974a3e3957fd1c1fa3082366d7fff6e428ddb55f074bf64876f8e8ad/zipp-3.6.0-py3-none-any.whl", hash = "sha256:9fe5ea21568a0a70e50f273397638d39b03353731e6cbbb3fd8502a33fec40bc"}, +] From df42850cc9602cf09c21f7ecd2a41f06dbced517 Mon Sep 17 00:00:00 2001 From: Nish Sinha Date: Tue, 18 Oct 2022 09:22:27 -0400 Subject: [PATCH 10/10] Refactor parse_pep621_dependencies to parse optional-dependencies --- python/helpers/lib/parser.py | 63 ++++++++++++------- .../pyproject_files_parser_spec.rb | 10 +++ .../optional_dependencies.toml | 21 +++++++ 3 files changed, 72 insertions(+), 22 deletions(-) create mode 100644 python/spec/fixtures/pyproject_files/optional_dependencies.toml diff --git a/python/helpers/lib/parser.py b/python/helpers/lib/parser.py index c9185386952..1b1cc249ddc 100644 --- a/python/helpers/lib/parser.py +++ b/python/helpers/lib/parser.py @@ -21,32 +21,51 @@ def parse_pep621_dependencies(pyproject_path): - dependencies = toml.load(pyproject_path)['project']['dependencies'] + project_toml = toml.load(pyproject_path)['project'] + + def parse_toml_section_pep621_dependencies(pyproject_path, dependencies): + requirement_packages = [] + + def version_from_req(specifier_set): + if (len(specifier_set) == 1 and + next(iter(specifier_set)).operator in {"==", "==="}): + return next(iter(specifier_set)).version + + for dependency in dependencies: + try: + req = Requirement(dependency) + except InvalidRequirement as e: + print(json.dumps({"error": repr(e)})) + exit(1) + else: + requirement_packages.append({ + "name": req.name, + "version": version_from_req(req.specifier), + "markers": str(req.marker) or None, + "file": pyproject_path, + "requirement": str(req.specifier), + "extras": sorted(list(req.extras)) + }) - requirement_packages = [] + return requirement_packages - def version_from_req(specifier_set): - if (len(specifier_set) == 1 and - next(iter(specifier_set)).operator in {"==", "==="}): - return next(iter(specifier_set)).version + dependencies = parse_toml_section_pep621_dependencies( + pyproject_path, + project_toml['dependencies'] + ) - for dependency in dependencies: - try: - req = Requirement(dependency) - except InvalidRequirement as e: - print(json.dumps({"error": repr(e)})) - exit(1) - else: - requirement_packages.append({ - "name": req.name, - "version": version_from_req(req.specifier), - "markers": str(req.marker) or None, - "file": pyproject_path, - "requirement": str(req.specifier), - "extras": sorted(list(req.extras)) - }) + if 'optional-dependencies' in project_toml: + optional_dependencies_toml = project_toml['optional-dependencies'] - return json.dumps({"result": requirement_packages}) + for group in optional_dependencies_toml: + group_dependencies = parse_toml_section_pep621_dependencies( + pyproject_path, + optional_dependencies_toml[group] + ) + + dependencies.extend(group_dependencies) + + return json.dumps({"result": dependencies}) def parse_requirements(directory): diff --git a/python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb b/python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb index 28cb0ba78c2..0fa36d3d63e 100644 --- a/python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb +++ b/python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb @@ -289,5 +289,15 @@ its(:length) { is_expected.to eq(0) } end + + context "with optional dependencies" do + let(:pyproject_fixture_name) { "optional_dependencies.toml" } + + subject(:dependencies) { parser.dependency_set.dependencies } + + # fixture has 1 runtime dependency, plus 4 optional dependencies, but one + # is ignored because it has markers + its(:length) { is_expected.to eq(4) } + end end end diff --git a/python/spec/fixtures/pyproject_files/optional_dependencies.toml b/python/spec/fixtures/pyproject_files/optional_dependencies.toml new file mode 100644 index 00000000000..a7c4bf33f69 --- /dev/null +++ b/python/spec/fixtures/pyproject_files/optional_dependencies.toml @@ -0,0 +1,21 @@ +[build-system] +requires = ["flit_core >=3.2,<4"] +build-backend = "flit_core.buildapi" + +[project] +name = "pkgtest" +authors = [{name = "Sample", email = "sample.project@example.org"}] +license = {file = "LICENSE"} +classifiers = ["License :: OSI Approved :: MIT License"] +dynamic = ["version", "description"] +dependencies = [ + "ansys-templates==0.3.0", +] + +[project.optional-dependencies] +socks = [ 'PySocks >= 1.5.6, != 1.5.7, < 2' ] +tests = [ + 'ddt >= 1.2.2, < 2', + 'pytest < 6', + 'mock >= 1.0.1, < 4; python_version < "3.4"', +]