Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Pyproject.toml optional-dependencies #5920

Closed
wants to merge 10 commits into from
51 changes: 51 additions & 0 deletions python/helpers/lib/parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,62 @@
install_req_from_line,
install_req_from_parsed_requirement,
)

from packaging.requirements import InvalidRequirement, Requirement
import toml

# Inspired by pips internal check:
# https://github.com/pypa/pip/blob/0bb3ac87f5bb149bd75cceac000844128b574385/src/pip/_internal/req/req_file.py#L35
COMMENT_RE = re.compile(r'(^|\s+)#.*$')


def parse_pep621_dependencies(pyproject_path):
DEPENDENCY_TYPES = ['dependencies', 'optional-dependencies']

def version_from_req(specifier_set):
if (len(specifier_set) == 1 and
next(iter(specifier_set)).operator in {"==", "==="}):
return next(iter(specifier_set)).version

def flatten(foo):
for x in foo:
if hasattr(x, '__iter__') and not isinstance(x, str):
for y in flatten(x):
yield y
else:
yield x

def parse_dependencies(dependency_type):
dependencies = []
requirement_packages = []

try:
project = toml.load(pyproject_path)['project']
if dependency_type in project:
dependencies = flatten(project[dependency_type])
for dependency in dependencies:
req = Requirement(dependency)
requirement_packages.append({
"name": req.name,
"version": version_from_req(req.specifier),
"markers": str(req.marker) or None,
"file": pyproject_path,
"requirement": str(req.specifier),
"requirement_type": dependency_type,
"extras": sorted(list(req.extras))
})
except KeyError as e:
print(json.dumps({"error": repr(e)}))
exit(1)
except InvalidRequirement as e:
print(json.dumps({"error": repr(e)}))
exit(1)
else:
return requirement_packages

parsed_dependencies = list(map(parse_dependencies, DEPENDENCY_TYPES))
return json.dumps({"result": parsed_dependencies})

def parse_requirements(directory):
# Parse the requirements.txt
requirement_packages = []
Expand Down
2 changes: 2 additions & 0 deletions python/helpers/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
print(parser.parse_requirements(args["args"][0]))
elif args["function"] == "parse_setup":
print(parser.parse_setup(args["args"][0]))
elif args["function"] == "parse_pep621_dependencies":
print(parser.parse_pep621_dependencies(args["args"][0]))
elif args["function"] == "get_dependency_hash":
print(hasher.get_dependency_hash(*args["args"]))
elif args["function"] == "get_pipfile_hash":
Expand Down
10 changes: 5 additions & 5 deletions python/lib/dependabot/python/file_fetcher.rb
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
require "dependabot/file_fetchers"
require "dependabot/file_fetchers/base"
require "dependabot/python/requirement_parser"
require "dependabot/python/file_parser/poetry_files_parser"
require "dependabot/python/file_parser/pyproject_files_parser"
require "dependabot/errors"

module Dependabot
Expand All @@ -24,7 +24,7 @@ def self.required_files_in?(filenames)
# If this repo is using a Pipfile return true
return true if filenames.include?("Pipfile")

# If this repo is using Poetry return true
# If this repo is using pyproject.toml return true
return true if filenames.include?("pyproject.toml")

return true if filenames.include?("setup.py")
Expand Down Expand Up @@ -296,8 +296,8 @@ def fetch_path_setup_file(path, allow_pyproject: false)
fetch_submodules: true
).tap { |f| f.support_file = true }
rescue Dependabot::DependencyFileNotFound
# For Poetry projects attempt to fetch a pyproject.toml at the
# given path instead of a setup.py. We do not require a
# For projects with pyproject.toml attempt to fetch a pyproject.toml
# at the given path instead of a setup.py. We do not require a
# setup.py to be present, so if none can be found, simply return
return [] unless allow_pyproject

Expand Down Expand Up @@ -395,7 +395,7 @@ def poetry_path_setup_file_paths
return [] unless pyproject

paths = []
Dependabot::Python::FileParser::PoetryFilesParser::POETRY_DEPENDENCY_TYPES.each do |dep_type|
Dependabot::Python::FileParser::PyprojectFilesParser::POETRY_DEPENDENCY_TYPES.each do |dep_type|
next unless parsed_pyproject.dig("tool", "poetry", dep_type)

parsed_pyproject.dig("tool", "poetry", dep_type).each do |_, req|
Expand Down
34 changes: 5 additions & 29 deletions python/lib/dependabot/python/file_parser.rb
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
# frozen_string_literal: true

require "toml-rb"
require "dependabot/dependency"
require "dependabot/file_parsers"
require "dependabot/file_parsers/base"
Expand All @@ -15,11 +14,9 @@ module Dependabot
module Python
class FileParser < Dependabot::FileParsers::Base
require_relative "file_parser/pipfile_files_parser"
require_relative "file_parser/poetry_files_parser"
require_relative "file_parser/pyproject_files_parser"
require_relative "file_parser/setup_file_parser"

POETRY_DEPENDENCY_TYPES =
%w(tool.poetry.dependencies tool.poetry.dev-dependencies).freeze
DEPENDENCY_GROUP_KEYS = [
{
pipfile: "packages",
Expand All @@ -42,7 +39,7 @@ def parse
dependency_set = DependencySet.new

dependency_set += pipenv_dependencies if pipfile
dependency_set += poetry_dependencies if using_poetry?
dependency_set += pyproject_file_dependencies if pyproject
dependency_set += requirement_dependencies if requirement_files.any?
dependency_set += setup_file_dependencies if setup_file || setup_cfg_file

Expand All @@ -62,9 +59,9 @@ def pipenv_dependencies
dependency_set
end

def poetry_dependencies
@poetry_dependencies ||=
PoetryFilesParser.
def pyproject_file_dependencies
@pyproject_file_dependencies ||=
PyprojectFilesParser.
new(dependency_files: dependency_files).
dependency_set
end
Expand Down Expand Up @@ -105,18 +102,6 @@ def group_from_filename(filename)
end
end

def included_in_pipenv_deps?(dep_name)
return false unless pipfile

pipenv_dependencies.dependencies.map(&:name).include?(dep_name)
end

def included_in_poetry_deps?(dep_name)
return false unless using_poetry?

poetry_dependencies.dependencies.map(&:name).include?(dep_name)
end

def blocking_marker?(dep)
return false if dep["markers"] == "None"
return true if dep["markers"].include?("<")
Expand Down Expand Up @@ -215,15 +200,6 @@ def pipfile_lock
@pipfile_lock ||= get_original_file("Pipfile.lock")
end

def using_poetry?
return false unless pyproject
return true if poetry_lock || pyproject_lock

!TomlRB.parse(pyproject.content).dig("tool", "poetry").nil?
rescue TomlRB::ParseError, TomlRB::ValueOverwriteError
raise Dependabot::DependencyFileNotParseable, pyproject.path
end

def output_file_regex(filename)
"--output-file[=\s]+#{Regexp.escape(filename)}(?:\s|$)"
end
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
module Dependabot
module Python
class FileParser
class PoetryFilesParser
class PyprojectFilesParser
POETRY_DEPENDENCY_TYPES = %w(dependencies dev-dependencies).freeze

# https://python-poetry.org/docs/dependency-specification/
Expand All @@ -25,7 +25,7 @@ def initialize(dependency_files:)
def dependency_set
dependency_set = Dependabot::FileParsers::Base::DependencySet.new

dependency_set += pyproject_dependencies
dependency_set += pyproject_dependencies if using_poetry? || using_pep621?
dependency_set += lockfile_dependencies if lockfile

dependency_set
Expand All @@ -36,6 +36,14 @@ def dependency_set
attr_reader :dependency_files

def pyproject_dependencies
if using_poetry?
poetry_dependencies
else
pep621_dependencies
end
end

def poetry_dependencies
dependencies = Dependabot::FileParsers::Base::DependencySet.new

POETRY_DEPENDENCY_TYPES.each do |type|
Expand All @@ -59,6 +67,38 @@ def pyproject_dependencies
dependencies
end

def pep621_dependencies
dependencies = Dependabot::FileParsers::Base::DependencySet.new

parsed_pep621_dependencies.each do |dep|
# If a requirement has a `<` or `<=` marker then updating it is
# probably blocked. Ignore it.
next if dep["markers"].include?("<")

# If no requirement, don't add it
next if dep["requirement"].empty?

dependencies <<
Dependency.new(
name: normalised_name(dep["name"], dep["extras"]),
version: dep["version"]&.include?("*") ? nil : dep["version"],
requirements: [{
requirement: dep["requirement"],
file: Pathname.new(dep["file"]).cleanpath.to_path,
source: nil,
groups: [dep["requirement_type"]]
}],
package_manager: "pip"
)
end

dependencies
end

def normalised_name(name, extras)
NameNormaliser.normalise_including_extras(name, extras)
end

# @param req can be an Array, Hash or String that represents the constraints for a dependency
def parse_requirements_from(req, type)
[req].flatten.compact.filter_map do |requirement|
Expand All @@ -75,6 +115,14 @@ def parse_requirements_from(req, type)
end
end

def using_poetry?
!parsed_pyproject.dig("tool", "poetry").nil?
end

def using_pep621?
!parsed_pyproject.dig("project", "dependencies").nil?
end

# Create a DependencySet where each element has no requirement. Any
# requirements will be added when combining the DependencySet with
# other DependencySets.
Expand Down Expand Up @@ -146,6 +194,24 @@ def lockfile
poetry_lock || pyproject_lock
end

def parsed_pep621_dependencies
SharedHelpers.in_a_temporary_directory do
write_temporary_pyproject

SharedHelpers.run_helper_subprocess(
command: "pyenv exec python #{NativeHelpers.python_helper_path}",
function: "parse_pep621_dependencies",
args: [pyproject.name]
)
end
end

def write_temporary_pyproject
path = pyproject.name
FileUtils.mkdir_p(Pathname.new(path).dirname)
File.write(path, pyproject.content)
end

def parsed_lockfile
return parsed_poetry_lock if poetry_lock
return parsed_pyproject_lock if pyproject_lock
Expand Down
15 changes: 14 additions & 1 deletion python/lib/dependabot/python/file_updater.rb
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# frozen_string_literal: true

require "toml-rb"
require "dependabot/file_updaters"
require "dependabot/file_updaters/base"
require "dependabot/shared_helpers"
Expand Down Expand Up @@ -61,7 +62,13 @@ def resolver_type
# Otherwise, this is a top-level dependency, and we can figure out
# which resolver to use based on the filename of its requirements
return :pipfile if changed_req_files.any?("Pipfile")
return :poetry if changed_req_files.any?("pyproject.toml")

if changed_req_files.any?("pyproject.toml")
return :poetry if poetry_based?

return :requirements
end

return :pip_compile if changed_req_files.any? { |f| f.end_with?(".in") }

:requirements
Expand Down Expand Up @@ -119,6 +126,12 @@ def check_required_files
raise "Missing required files!"
end

def poetry_based?
return false unless pyproject

!TomlRB.parse(pyproject.content).dig("tool", "poetry").nil?
end

def pipfile
@pipfile ||= get_original_file("Pipfile")
end
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ def freeze_dependencies_being_updated(pyproject_content)
end

def lock_declaration_to_new_version!(poetry_object, dep)
Dependabot::Python::FileParser::PoetryFilesParser::POETRY_DEPENDENCY_TYPES.each do |type|
Dependabot::Python::FileParser::PyprojectFilesParser::POETRY_DEPENDENCY_TYPES.each do |type|
names = poetry_object[type]&.keys || []
pkg_name = names.find { |nm| normalise(nm) == dep.name }
next unless pkg_name
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ def freeze_top_level_dependencies_except(dependencies)
poetry_object = pyproject_object["tool"]["poetry"]
excluded_names = dependencies.map(&:name) + ["python"]

Dependabot::Python::FileParser::PoetryFilesParser::POETRY_DEPENDENCY_TYPES.each do |key|
Dependabot::Python::FileParser::PyprojectFilesParser::POETRY_DEPENDENCY_TYPES.each do |key|
next unless poetry_object[key]

source_types = %w(directory file url)
Expand Down
Loading