-
Notifications
You must be signed in to change notification settings - Fork 36
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Make conda return Lockfile, or Manifest, not both in one response. #467
Changes from all commits
5a5548d
1e6ba93
5726e0a
3ada142
92978e1
ad249ba
6155486
276542c
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,67 +1,53 @@ | ||
require 'json' | ||
require "json" | ||
|
||
module Bibliothecary | ||
module Parsers | ||
class Conda | ||
include Bibliothecary::Analyser | ||
FILE_KINDS = %w[manifest lockfile] | ||
|
||
def self.mapping | ||
{ | ||
match_filename("environment.yml") => { | ||
kind: FILE_KINDS | ||
parser: :parse_conda, | ||
kind: "manifest", | ||
}, | ||
match_filename("environment.yaml") => { | ||
kind: FILE_KINDS | ||
} | ||
parser: :parse_conda, | ||
kind: "manifest", | ||
}, | ||
match_filename("environment.yml.lock") => { | ||
parser: :parse_conda_lockfile, | ||
kind: "lockfile", | ||
}, | ||
match_filename("environment.yaml.lock") => { | ||
parser: :parse_conda_lockfile, | ||
kind: "lockfile", | ||
}, | ||
} | ||
end | ||
|
||
# Overrides Analyser.analyse_contents_from_info | ||
def self.analyse_contents_from_info(info) | ||
[parse_conda(info), parse_pip(info)].flatten.compact | ||
rescue Bibliothecary::RemoteParsingError => e | ||
Bibliothecary::Analyser::create_error_analysis(platform_name, info.relative_path, "runtime", e.message) | ||
rescue Psych::SyntaxError => e | ||
Bibliothecary::Analyser::create_error_analysis(platform_name, info.relative_path, "runtime", e.message) | ||
end | ||
|
||
private | ||
|
||
def self.parse_conda(info) | ||
results = call_conda_parser_web(info.contents) | ||
FILE_KINDS.map do |kind| | ||
Bibliothecary::Analyser.create_analysis( | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. SEE YA Explicit Creation |
||
"conda", | ||
info.relative_path, | ||
kind, | ||
results[kind.to_sym].map { |dep| dep.slice(:name, :requirement).merge(type: "runtime") } | ||
) | ||
end | ||
dependencies = call_conda_parser_web(info, :manifest)[:manifest] | ||
dependencies.map { |dep| dep.merge(type: "runtime") } | ||
end | ||
|
||
def self.parse_pip(info) | ||
dependencies = YAML.safe_load(info.contents)["dependencies"] | ||
pip = dependencies.find { |dep| dep.is_a?(Hash) && dep["pip"]} | ||
return unless pip | ||
|
||
Bibliothecary::Analyser.create_analysis( | ||
"pypi", | ||
info.relative_path, | ||
"manifest", | ||
Pypi.parse_requirements_txt(pip["pip"].join("\n")) | ||
) | ||
def self.parse_conda_lockfile(info) | ||
dependencies = call_conda_parser_web(info, :lockfile)[:lockfile] | ||
dependencies.map { |dep| dep.merge(type: "runtime") } | ||
end | ||
|
||
def self.call_conda_parser_web(file_contents) | ||
private_class_method def self.call_conda_parser_web(file_contents, kind) | ||
host = Bibliothecary.configuration.conda_parser_host | ||
response = Typhoeus.post( | ||
"#{host}/parse", | ||
headers: { | ||
ContentType: 'multipart/form-data' | ||
ContentType: "multipart/form-data", | ||
}, | ||
# hardcoding `environment.yml` to send to `conda.libraries.io`, downside is logs will always show `environment.yml` there | ||
body: {file: file_contents, filename: 'environment.yml'} | ||
body: { | ||
file: file_contents, | ||
# Unfortunately we do not get the filename in the mapping parsers, so hardcoding the file name depending on the kind | ||
filename: kind == "manifest" ? "environment.yml" : "environment.yml.lock", | ||
} | ||
) | ||
raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{host}/parse", response.response_code) unless response.success? | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,12 +1,12 @@ | ||
require 'spec_helper' | ||
require "spec_helper" | ||
|
||
describe Bibliothecary::Parsers::Conda do | ||
it 'has a platform name' do | ||
expect(described_class.platform_name).to eq('conda') | ||
it "has a platform name" do | ||
expect(described_class.platform_name).to eq("conda") | ||
end | ||
|
||
it 'parses dependencies from environment.yml', :vcr do | ||
expect(described_class.analyse_contents('environment.yml', load_fixture('environment.yml'))).to eq([ | ||
it "parses dependencies from environment.yml", :vcr do | ||
expect(described_class.analyse_contents("environment.yml", load_fixture("environment.yml"))).to eq( | ||
{ | ||
:platform=>"conda", | ||
:path=>"environment.yml", | ||
|
@@ -17,116 +17,98 @@ | |
{:name=>"ncurses", :requirement=>"6.1", :type=>"runtime"}, | ||
{:name=>"numpy", :requirement=>"1.16.4", :type=>"runtime"}, | ||
{:name=>"openssl", :requirement=>"1.1.1c", :type=>"runtime"}, | ||
{:name=>"pip", :requirement=>"19.2.2", :type=>"runtime"}, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. no version because we're not resolving the version now for manifests, just returning as is. this example file has pip and setuptools without versions |
||
{:name=>"pip", :requirement=>"", :type=>"runtime"}, | ||
{:name=>"python", :requirement=>"3.7.3", :type=>"runtime"}, | ||
{:name=>"readline", :requirement=>"7.0", :type=>"runtime"}, | ||
{:name=>"setuptools", :requirement=>"41.0.1", :type=>"runtime"}, | ||
{:name=>"setuptools", :requirement=>"", :type=>"runtime"}, | ||
{:name=>"sqlite", :requirement=>"3.29.0", :type=>"runtime"} | ||
], | ||
kind: "manifest", | ||
success: true | ||
}, | ||
} | ||
) | ||
end | ||
|
||
it "parses dependencies from environment.yml.lock", :vcr do | ||
expect(described_class.analyse_contents("environment.yml.lock", load_fixture("environment.yml"))).to eq( | ||
{ | ||
:platform=>"conda", | ||
:path=>"environment.yml", | ||
:path=>"environment.yml.lock", | ||
:dependencies=>[ | ||
{:name=>"_libgcc_mutex", :requirement=>"0.1", :type=>"runtime"}, | ||
{:name=>"beautifulsoup4", :requirement=>"4.7.1", :type=>"runtime"}, | ||
{:name=>"biopython", :requirement=>"1.74", :type=>"runtime"}, | ||
{:name=>"blas", :requirement=>"1.0", :type=>"runtime"}, | ||
{:name=>"ca-certificates", :requirement=>"2019.5.15", :type=>"runtime"}, | ||
{:name=>"certifi", :requirement=>"2019.6.16", :type=>"runtime"}, | ||
{:name=>"intel-openmp", :requirement=>"2019.4", :type=>"runtime"}, | ||
{:name=>"libedit", :requirement=>"3.1.20181209", :type=>"runtime"}, | ||
{:name=>"libffi", :requirement=>"3.2.1", :type=>"runtime"}, | ||
{:name=>"libgcc-ng", :requirement=>"9.1.0", :type=>"runtime"}, | ||
{:name=>"libgfortran-ng", :requirement=>"7.3.0", :type=>"runtime"}, | ||
{:name=>"libstdcxx-ng", :requirement=>"9.1.0", :type=>"runtime"}, | ||
{:name=>"mkl", :requirement=>"2019.4", :type=>"runtime"}, | ||
{:name=>"mkl-service", :requirement=>"2.0.2", :type=>"runtime"}, | ||
{:name=>"mkl_fft", :requirement=>"1.0.14", :type=>"runtime"}, | ||
{:name=>"mkl_random", :requirement=>"1.0.2", :type=>"runtime"}, | ||
{:name=>"ncurses", :requirement=>"6.1", :type=>"runtime"}, | ||
{:name=>"numpy", :requirement=>"1.16.4", :type=>"runtime"}, | ||
{:name=>"numpy-base", :requirement=>"1.16.4", :type=>"runtime"}, | ||
{:name=>"openssl", :requirement=>"1.1.1c", :type=>"runtime"}, | ||
{:name=>"pip", :requirement=>"19.2.2", :type=>"runtime"}, | ||
{:name=>"python", :requirement=>"3.7.3", :type=>"runtime"}, | ||
{:name=>"readline", :requirement=>"7.0", :type=>"runtime"}, | ||
{:name=>"setuptools", :requirement=>"41.0.1", :type=>"runtime"}, | ||
{:name=>"six", :requirement=>"1.12.0", :type=>"runtime"}, | ||
{:name=>"soupsieve", :requirement=>"1.9.2", :type=>"runtime"}, | ||
{:name=>"sqlite", :requirement=>"3.29.0", :type=>"runtime"}, | ||
{:name=>"tk", :requirement=>"8.6.8", :type=>"runtime"}, | ||
{:name=>"wheel", :requirement=>"0.33.4", :type=>"runtime"}, | ||
{:name=>"xz", :requirement=>"5.2.4", :type=>"runtime"}, | ||
{:name=>"zlib", :requirement=>"1.2.11", :type=>"runtime"} | ||
{:name=>"_libgcc_mutex", :requirement=>"0.1", :type=>"runtime"}, | ||
{:name=>"beautifulsoup4", :requirement=>"4.7.1", :type=>"runtime"}, | ||
{:name=>"biopython", :requirement=>"1.74", :type=>"runtime"}, | ||
{:name=>"blas", :requirement=>"1.0", :type=>"runtime"}, | ||
{:name=>"ca-certificates", :requirement=>"2019.8.28", :type=>"runtime"}, | ||
{:name=>"certifi", :requirement=>"2019.6.16", :type=>"runtime"}, | ||
{:name=>"intel-openmp", :requirement=>"2019.4", :type=>"runtime"}, | ||
{:name=>"libedit", :requirement=>"3.1.20181209", :type=>"runtime"}, | ||
{:name=>"libffi", :requirement=>"3.2.1", :type=>"runtime"}, | ||
{:name=>"libgcc-ng", :requirement=>"9.1.0", :type=>"runtime"}, | ||
{:name=>"libgfortran-ng", :requirement=>"7.3.0", :type=>"runtime"}, | ||
{:name=>"libstdcxx-ng", :requirement=>"9.1.0", :type=>"runtime"}, | ||
{:name=>"mkl", :requirement=>"2019.4", :type=>"runtime"}, | ||
{:name=>"mkl-service", :requirement=>"2.3.0", :type=>"runtime"}, | ||
{:name=>"mkl_fft", :requirement=>"1.0.14", :type=>"runtime"}, | ||
{:name=>"mkl_random", :requirement=>"1.1.0", :type=>"runtime"}, | ||
{:name=>"ncurses", :requirement=>"6.1", :type=>"runtime"}, | ||
{:name=>"numpy", :requirement=>"1.16.4", :type=>"runtime"}, | ||
{:name=>"numpy-base", :requirement=>"1.16.4", :type=>"runtime"}, | ||
{:name=>"openssl", :requirement=>"1.1.1c", :type=>"runtime"}, | ||
{:name=>"pip", :requirement=>"19.2.3", :type=>"runtime"}, | ||
{:name=>"python", :requirement=>"3.7.3", :type=>"runtime"}, | ||
{:name=>"readline", :requirement=>"7.0", :type=>"runtime"}, | ||
{:name=>"setuptools", :requirement=>"41.2.0", :type=>"runtime"}, | ||
{:name=>"six", :requirement=>"1.12.0", :type=>"runtime"}, | ||
{:name=>"soupsieve", :requirement=>"1.9.3", :type=>"runtime"}, | ||
{:name=>"sqlite", :requirement=>"3.29.0", :type=>"runtime"}, | ||
{:name=>"tk", :requirement=>"8.6.8", :type=>"runtime"}, | ||
{:name=>"wheel", :requirement=>"0.33.6", :type=>"runtime"}, | ||
{:name=>"xz", :requirement=>"5.2.4", :type=>"runtime"}, | ||
{:name=>"zlib", :requirement=>"1.2.11", :type=>"runtime"} | ||
], | ||
kind: "lockfile", | ||
success: true | ||
} | ||
] | ||
) | ||
end | ||
|
||
it 'parses dependencies from environment.yml with pip', :vcr do | ||
expect(described_class.analyse_contents('conda_with_pip/environment.yml', load_fixture('conda_with_pip/environment.yml'))).to eq([ | ||
{ | ||
:platform=>"conda", | ||
:path=>"conda_with_pip/environment.yml", | ||
:dependencies=>[ | ||
{:name=>"pip", :requirement=>"19.2.2", :type=>"runtime"}, | ||
{:name=>"sqlite", :requirement=>"3.29.0", :type=>"runtime"} | ||
], | ||
kind: "manifest", | ||
success: true | ||
}, | ||
{ | ||
:platform=>"conda", | ||
:path=>"conda_with_pip/environment.yml", | ||
:dependencies=>[ | ||
{:name=>"_libgcc_mutex", :requirement=>"0.1", :type=>"runtime"}, | ||
{:name=>"ca-certificates", :requirement=>"2019.5.15", :type=>"runtime"}, | ||
{:name=>"certifi", :requirement=>"2019.6.16", :type=>"runtime"}, | ||
{:name=>"libedit", :requirement=>"3.1.20181209", :type=>"runtime"}, | ||
{:name=>"libffi", :requirement=>"3.2.1", :type=>"runtime"}, | ||
{:name=>"libgcc-ng", :requirement=>"9.1.0", :type=>"runtime"}, | ||
{:name=>"libstdcxx-ng", :requirement=>"9.1.0", :type=>"runtime"}, | ||
{:name=>"ncurses", :requirement=>"6.1", :type=>"runtime"}, | ||
{:name=>"openssl", :requirement=>"1.1.1c", :type=>"runtime"}, | ||
{:name=>"pip", :requirement=>"19.2.2", :type=>"runtime"}, | ||
{:name=>"python", :requirement=>"3.7.4", :type=>"runtime"}, | ||
{:name=>"readline", :requirement=>"7.0", :type=>"runtime"}, | ||
{:name=>"setuptools", :requirement=>"41.0.1", :type=>"runtime"}, | ||
{:name=>"sqlite", :requirement=>"3.29.0", :type=>"runtime"}, | ||
{:name=>"tk", :requirement=>"8.6.8", :type=>"runtime"}, | ||
{:name=>"wheel", :requirement=>"0.33.4", :type=>"runtime"}, | ||
{:name=>"xz", :requirement=>"5.2.4", :type=>"runtime"}, | ||
{:name=>"zlib", :requirement=>"1.2.11", :type=>"runtime"}, | ||
], | ||
kind: "lockfile", | ||
success: true | ||
}, | ||
{ | ||
:platform=>"pypi", | ||
:path=>"conda_with_pip/environment.yml", | ||
:dependencies=>[ | ||
{:name=>"urllib3", :requirement=>"*", :type=>"runtime"}, | ||
{:name=>"Django", :requirement=>"==2.0.0", :type=>"runtime"}, | ||
|
||
], | ||
kind: "manifest", | ||
success: true | ||
it "parses dependencies from environment.yml.lock with pip", :vcr do | ||
expect(described_class.analyse_contents("conda_with_pip/environment.yml.lock", load_fixture("conda_with_pip/environment.yml"))).to eq( | ||
{ | ||
:platform=>"conda", | ||
:path=>"conda_with_pip/environment.yml.lock", | ||
:dependencies=>[ | ||
{:name=>"_libgcc_mutex", :requirement=>"0.1", :type=>"runtime"}, | ||
{:name=>"ca-certificates", :requirement=>"2019.8.28", :type=>"runtime"}, | ||
{:name=>"certifi", :requirement=>"2019.9.11", :type=>"runtime"}, | ||
{:name=>"libedit", :requirement=>"3.1.20181209", :type=>"runtime"}, | ||
{:name=>"libffi", :requirement=>"3.2.1", :type=>"runtime"}, | ||
{:name=>"libgcc-ng", :requirement=>"9.1.0", :type=>"runtime"}, | ||
{:name=>"libstdcxx-ng", :requirement=>"9.1.0", :type=>"runtime"}, | ||
{:name=>"ncurses", :requirement=>"6.1", :type=>"runtime"}, | ||
{:name=>"openssl", :requirement=>"1.1.1d", :type=>"runtime"}, | ||
{:name=>"pip", :requirement=>"19.2.3", :type=>"runtime"}, | ||
{:name=>"python", :requirement=>"3.7.4", :type=>"runtime"}, | ||
{:name=>"readline", :requirement=>"7.0", :type=>"runtime"}, | ||
{:name=>"setuptools", :requirement=>"41.2.0", :type=>"runtime"}, | ||
{:name=>"sqlite", :requirement=>"3.29.0", :type=>"runtime"}, | ||
{:name=>"tk", :requirement=>"8.6.8", :type=>"runtime"}, | ||
{:name=>"wheel", :requirement=>"0.33.6", :type=>"runtime"}, | ||
{:name=>"xz", :requirement=>"5.2.4", :type=>"runtime"}, | ||
{:name=>"zlib", :requirement=>"1.2.11", :type=>"runtime"} | ||
], | ||
kind: "lockfile", | ||
success: true | ||
} | ||
] | ||
) | ||
) | ||
end | ||
|
||
it 'matches valid manifest filepaths' do | ||
expect(described_class.match?('environment.yml')).to be_truthy | ||
it "matches valid manifest filepaths" do | ||
expect(described_class.match?("environment.yml")).to be_truthy | ||
end | ||
|
||
it "doesn't match invalid manifest filepaths" do | ||
expect(described_class.match?('test/foo/aenvironment.yml')).to be_falsey | ||
expect(described_class.match?("test/foo/aenvironment.yml")).to be_falsey | ||
end | ||
end | ||
end |
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
GOOD BYE OVERRIDE!