Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Extract index and gemspec fetching into Bundler::Fetcher

  • Loading branch information...
commit 9eeebb3229694abe530571cf4c4f867d915a2840 1 parent 80f56e1
André Arko indirect authored hone committed
1  lib/bundler.rb
View
@@ -12,6 +12,7 @@ module Bundler
autoload :Dependency, 'bundler/dependency'
autoload :Dsl, 'bundler/dsl'
autoload :Environment, 'bundler/environment'
+ autoload :Fetcher, 'bundler/fetcher'
autoload :GemHelper, 'bundler/gem_helper'
autoload :Graph, 'bundler/graph'
autoload :Index, 'bundler/index'
80 lib/bundler/fetcher.rb
View
@@ -0,0 +1,80 @@
+require 'uri'
+require 'net/http/persistent'
+
+module Bundler
+ class Fetcher
+ def initialize(remote_uri)
+ @remote_uri = remote_uri
+ @@connection ||= Net::HTTP::Persistent.new
+ end
+
+ # fetch a gem specification
+ def fetch_spec(spec)
+ spec = spec - [nil, 'ruby', '']
+ spec_file_name = "#{spec.join '-'}.gemspec.rz"
+
+ uri = URI.parse("#{@remote_uri}#{Gem::MARSHAL_SPEC_DIR}#{spec_file_name}")
+ Bundler.ui.debug "Fetching spec from: #{uri}"
+
+ spec_rz = (uri.scheme == "file") ? Gem.read_binary(uri.path) : fetch(uri)
+ Marshal.load Gem.inflate(spec_rz)
+ end
+
+ # fetch index
+ def fetch_remote_specs(gem_names, full_dependency_list = [], last_spec_list = [], &blk)
+ return fetch_all_remote_specs(&blk) unless gem_names && @remote_uri.scheme != "file"
+
+ query_list = gem_names - full_dependency_list
+ Bundler.ui.debug "Query List: #{query_list.inspect}"
+ return {@remote_uri => last_spec_list}.each(&blk) if query_list.empty?
+
+ spec_list, deps_list = fetch_dependency_remote_specs(query_list, &blk)
+ returned_gems = spec_list.map {|spec| spec.first }.uniq
+
+ fetch_remote_specs(deps_list, full_dependency_list + returned_gems, spec_list + last_spec_list, &blk)
+ # fall back to the legacy index in the following cases
+ # 1.) Gemcutter Endpoint doesn't return a 200
+ # 2.) Marshal blob doesn't load properly
+ rescue OpenURI::HTTPError, TypeError
+ fetch_all_remote_specs(&blk)
+ end
+
+ private
+
+ def fetch(uri)
+ @@connection.request(uri).body
+ end
+
+ # fetch from Gemcutter Dependency Endpoint API
+ def fetch_dependency_remote_specs(gem_names, &blk)
+ marshalled_deps = @@connection.request("#{@remote_uri}api/v1/dependencies?gems=#{gem_names.join(",")}").body
+ gem_list = Marshal.load(marshalled_deps)
+
+ spec_list = gem_list.map do |s|
+ [s[:name], Gem::Version.new(s[:number]), s[:platform]]
+ end
+ deps_list = gem_list.map do |s|
+ s[:dependencies].collect {|d| d.first }
+ end.flatten.uniq
+
+ [spec_list, deps_list]
+ end
+
+ # fetch from modern index: specs.4.8.gz
+ def fetch_all_remote_specs(&blk)
+ Gem.sources = ["#{@remote_uri}"]
+ begin
+ # Fetch all specs, minus prerelease specs
+ Gem::SpecFetcher.new.list(true, false).each(&blk)
+ # Then fetch the prerelease specs
+ begin
+ Gem::SpecFetcher.new.list(false, true).each(&blk)
+ rescue Gem::RemoteFetcher::FetchError
+ Bundler.ui.warn "Could not fetch prerelease specs from #{self}"
+ end
+ rescue Gem::RemoteFetcher::FetchError
+ Bundler.ui.warn "Could not reach #{self}"
+ end
+ end
+ end
+end
14 lib/bundler/remote_specification.rb
View
@@ -12,11 +12,11 @@ class RemoteSpecification
attr_reader :name, :version, :platform
attr_accessor :source
- def initialize(name, version, platform, source_uri)
- @name = name
- @version = version
- @platform = platform
- @source_uri = source_uri
+ def initialize(name, version, platform, spec_fetcher)
+ @name = name
+ @version = version
+ @platform = platform
+ @spec_fetcher = spec_fetcher
end
# Needed before installs, since the arch matters then and quick
@@ -43,9 +43,7 @@ def __swap__(spec)
private
def _remote_specification
- @specification ||= begin
- Gem::SpecFetcher.new.fetch_spec([@name, @version, @platform], URI(@source_uri.to_s))
- end
+ @specification ||= @spec_fetcher.fetch_spec([@name, @version, @platform])
end
def method_missing(method, *args, &blk)
60 lib/bundler/source.rb
View
@@ -14,10 +14,11 @@ class Rubygems
def initialize(options = {})
@options = options
@remotes = (options["remotes"] || []).map { |r| normalize_uri(r) }
+ @fetchers = {}
@allow_remote = false
@allow_cached = false
- # Hardcode the paths for now
- @caches = [ Bundler.app_cache ] + Gem.path.map { |p| File.expand_path("#{p}/cache") }
+
+ @caches = [ Bundler.app_cache ] + Gem.path.map{ |p| File.expand_path("#{p}/cache") }
@spec_fetch_map = {}
end
@@ -210,15 +211,16 @@ def remote_specs(dependencies = nil)
remotes.each do |uri|
Bundler.ui.info "Fetching source index for #{uri}"
- Gem.sources = ["#{uri}"]
+
+ @fetchers[uri] = Bundler::Fetcher.new(uri)
gem_names =
if dependencies
dependencies.map {|d| d.name }
end
- fetch_remote_specs(gem_names, uri) do |n,v|
+ @fetchers[uri].fetch_remote_specs(gem_names, uri) do |n,v|
v.each do |name, version, platform|
next if name == 'bundler'
- spec = RemoteSpecification.new(name, version, platform, uri)
+ spec = RemoteSpecification.new(name, version, platform, @fetchers[uri])
spec.source = self
@spec_fetch_map[spec.full_name] = [spec, uri]
idx << spec
@@ -231,54 +233,6 @@ def remote_specs(dependencies = nil)
end
end
- def fetch_remote_specs(gem_names, uri, full_dependency_list = [], last_spec_list = [], &blk)
- require 'open-uri'
- return fetch_all_remote_specs(&blk) unless gem_names && uri.scheme != "file"
-
- query_list = gem_names - full_dependency_list
- Bundler.ui.debug "Query List: #{query_list.inspect}"
- return {uri => last_spec_list}.each(&blk) if query_list.empty?
-
- spec_list, deps_list = fetch_dependency_remote_specs(query_list, uri, &blk)
- returned_gems = spec_list.map {|spec| spec.first }.uniq
-
- fetch_remote_specs(deps_list, uri, full_dependency_list + returned_gems, spec_list + last_spec_list, &blk)
- # fall back to the legacy index in the following cases
- # 1.) Gemcutter Endpoint doesn't return a 200
- # 2.) Marshal blob doesn't load properly
- rescue OpenURI::HTTPError, TypeError
- fetch_all_remote_specs(&blk)
- end
-
- def fetch_dependency_remote_specs(gem_names, uri, &blk)
- marshalled_deps = open("#{uri}api/v1/dependencies?gems=#{gem_names.join(",")}").read
- gem_list = Marshal.load(marshalled_deps)
-
- spec_list = gem_list.map do |s|
- [s[:name], Gem::Version.new(s[:number]), s[:platform]]
- end
- deps_list = gem_list.map do |s|
- s[:dependencies].collect {|d| d.first }
- end.flatten.uniq
-
- [spec_list, deps_list]
- end
-
- def fetch_all_remote_specs(&blk)
- begin
- # Fetch all specs, minus prerelease specs
- Gem::SpecFetcher.new.list(true, false).each(&blk)
- # Then fetch the prerelease specs
- begin
- Gem::SpecFetcher.new.list(false, true).each(&blk)
- rescue Gem::RemoteFetcher::FetchError
- Bundler.ui.warn "Could not fetch prerelease specs from #{self}"
- end
- rescue Gem::RemoteFetcher::FetchError
- Bundler.ui.warn "Could not reach #{self}"
- end
- end
-
def download_gem_from_uri(spec, uri)
spec.fetch_platform
Please sign in to comment.
Something went wrong with that request. Please try again.