Merge pull request #4181 from rubygems/release/bundler_2.2.3_rubygems…

…_3.2.3

Prepare rubygems 3.2.3 and bundler 2.2.3

.github/workflows/install-rubygems.yml

@@ -62,6 +62,8 @@ jobs:
       - name: Check bundler install didn't hit the network
         run: if grep -q 'GET http' output.txt; then false; else true; fi
         working-directory: ./bundler
+      - name: Check rails can be installed
+        run: gem install rails
     timeout-minutes: 10
 
   install_rubygems_windows:

History.txt

@@ -1,3 +1,15 @@
+=== 3.2.3 / 2020-12-22
+
+Enhancements:
+
+* Fix misspellings in default API key name. Pull request #4177 by hsbt
+
+Bug fixes:
+
+* Respect `required_ruby_version` and `required_rubygems_version`
+  constraints when looking for `gem install` candidates. Pull request #4110
+  by deivid-rodriguez
+
 === 3.2.2 / 2020-12-17
 
 Bug fixes:

Manifest.txt

@@ -52,6 +52,7 @@ bundler/lib/bundler/cli/update.rb
 bundler/lib/bundler/cli/viz.rb
 bundler/lib/bundler/compact_index_client.rb
 bundler/lib/bundler/compact_index_client/cache.rb
+bundler/lib/bundler/compact_index_client/gem_parser.rb
 bundler/lib/bundler/compact_index_client/updater.rb
 bundler/lib/bundler/constants.rb
 bundler/lib/bundler/current_ruby.rb
@@ -412,6 +413,7 @@ lib/rubygems/requirement.rb
 lib/rubygems/resolver.rb
 lib/rubygems/resolver/activation_request.rb
 lib/rubygems/resolver/api_set.rb
+lib/rubygems/resolver/api_set/gem_parser.rb
 lib/rubygems/resolver/api_specification.rb
 lib/rubygems/resolver/best_set.rb
 lib/rubygems/resolver/composed_set.rb

bundler/CHANGELOG.md

@@ -1,3 +1,12 @@
+# 2.2.3 (December 22, 2020)
+
+## Bug fixes:
+
+  - Restore full compatibility with previous lockfiles [#4179](https://github.com/rubygems/rubygems/pull/4179)
+  - Add all matching variants with the same platform specificity to the lockfile [#4180](https://github.com/rubygems/rubygems/pull/4180)
+  - Fix bundler installing gems for a different platform when running in frozen mode and current platform not in the lockfile [#4172](https://github.com/rubygems/rubygems/pull/4172)
+  - Fix crash when `bundle exec`'ing to bundler [#4175](https://github.com/rubygems/rubygems/pull/4175)
+
 # 2.2.2 (December 17, 2020)
 
 ## Bug fixes:

bundler/lib/bundler.rb

@@ -212,13 +212,10 @@ def locked_gems
         end
     end
 
-    def locked_bundler_version
-      return nil unless defined?(@definition) && @definition
+    def most_specific_locked_platform?(platform)
+      return false unless defined?(@definition) && @definition
 
-      locked_gems = definition.locked_gems
-      return nil unless locked_gems
-
-      locked_gems.bundler_version
+      definition.most_specific_locked_platform == platform
     end
 
     def ruby_scope

bundler/lib/bundler/cli/update.rb

@@ -82,7 +82,7 @@ def run
           locked_spec = locked_info[:spec]
           new_spec = Bundler.definition.specs[name].first
           unless new_spec
-            if Bundler.rubygems.platforms.none? {|p| locked_spec.match_platform(p) }
+            unless locked_spec.match_platform(Bundler.local_platform)
               Bundler.ui.warn "Bundler attempted to update #{name} but it was not considered because it is for a different platform from the current one"
             end
 

bundler/lib/bundler/compact_index_client/cache.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "gem_parser"
+
 module Bundler
   class CompactIndexClient
     class Cache
@@ -92,19 +94,9 @@ def lines(path)
         header ? lines[header + 1..-1] : lines
       end
 
-      def parse_gem(string)
-        version_and_platform, rest = string.split(" ", 2)
-        version, platform = version_and_platform.split("-", 2)
-        dependencies, requirements = rest.split("|", 2).map {|s| s.split(",") } if rest
-        dependencies = dependencies ? dependencies.map {|d| parse_dependency(d) } : []
-        requirements = requirements ? requirements.map {|r| parse_dependency(r) } : []
-        [version, platform, dependencies, requirements]
-      end
-
-      def parse_dependency(string)
-        dependency = string.split(":")
-        dependency[-1] = dependency[-1].split("&") if dependency.size > 1
-        dependency
+      def parse_gem(line)
+        @dependency_parser ||= GemParser.new
+        @dependency_parser.parse(line)
       end
 
       def info_roots

bundler/lib/bundler/compact_index_client/gem_parser.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Bundler
+  class CompactIndexClient
+    if defined?(Gem::Resolver::APISet::GemParser)
+      GemParser = Gem::Resolver::APISet::GemParser
+    else
+      class GemParser
+        def parse(line)
+          version_and_platform, rest = line.split(" ", 2)
+          version, platform = version_and_platform.split("-", 2)
+          dependencies, requirements = rest.split("|", 2).map {|s| s.split(",") } if rest
+          dependencies = dependencies ? dependencies.map {|d| parse_dependency(d) } : []
+          requirements = requirements ? requirements.map {|d| parse_dependency(d) } : []
+          [version, platform, dependencies, requirements]
+        end
+
+        private
+
+        def parse_dependency(string)
+          dependency = string.split(":")
+          dependency[-1] = dependency[-1].split("&") if dependency.size > 1
+          dependency
+        end
+      end
+    end
+  end
+end

bundler/lib/bundler/definition.rb

@@ -118,7 +118,7 @@ def initialize(lockfile, dependencies, sources, unlock, ruby_version = nil, opti
       end
       @unlocking ||= @unlock[:ruby] ||= (!@locked_ruby_version ^ !@ruby_version)
 
-      add_current_platform unless Bundler.frozen_bundle?
+      add_current_platform unless current_ruby_platform_locked? || Bundler.frozen_bundle?
 
       converge_path_sources_to_gemspec_sources
       @path_changes = converge_paths
@@ -157,7 +157,7 @@ def resolve_with_cache!
     end
 
     def resolve_remotely!
-      raise "Specs already loaded" if @specs
+      return if @specs
       @remote = true
       sources.remote!
       specs
@@ -269,9 +269,8 @@ def resolve
           else
             # Run a resolve against the locally available gems
             Bundler.ui.debug("Found changes from the lockfile, re-resolving dependencies because #{change_reason}")
-            platforms_for_resolve = platforms.one? {|p| generic(p) == Gem::Platform::RUBY } ? platforms : platforms.reject{|p| p == Gem::Platform::RUBY }
-            expanded_dependencies = expand_dependencies(dependencies + metadata_dependencies, @remote, platforms_for_resolve.map {|p| generic(p) })
-            last_resolve.merge Resolver.resolve(expanded_dependencies, index, source_requirements, last_resolve, gem_version_promoter, additional_base_requirements_for_resolve, platforms_for_resolve)
+            expanded_dependencies = expand_dependencies(dependencies + metadata_dependencies, @remote)
+            last_resolve.merge Resolver.resolve(expanded_dependencies, index, source_requirements, last_resolve, gem_version_promoter, additional_base_requirements_for_resolve, platforms)
           end
 
         # filter out gems that _can_ be installed on multiple platforms, but don't need
@@ -507,15 +506,11 @@ def validate_ruby!
     end
 
     def validate_platforms!
-      return if @platforms.any? do |bundle_platform|
-        Bundler.rubygems.platforms.any? do |local_platform|
-          MatchPlatform.platforms_match?(bundle_platform, local_platform)
-        end
-      end
+      return if current_platform_locked?
 
       raise ProductionError, "Your bundle only supports platforms #{@platforms.map(&:to_s)} " \
-        "but your local platforms are #{Bundler.rubygems.platforms.map(&:to_s)}, and " \
-        "there's no compatible match between those two lists."
+        "but your local platform is #{Bundler.local_platform}. " \
+        "Add the current platform to the lockfile with `bundle lock --add-platform #{Bundler.local_platform}` and try again."
     end
 
     def add_platform(platform)
@@ -528,6 +523,12 @@ def remove_platform(platform)
       raise InvalidOption, "Unable to remove the platform `#{platform}` since the only platforms are #{@platforms.join ", "}"
     end
 
+    def most_specific_locked_platform
+      @platforms.min_by do |bundle_platform|
+        platform_specificity_match(bundle_platform, local_platform)
+      end
+    end
+
     def find_resolved_spec(current_spec)
       specs.find_by_name_and_platform(current_spec.name, current_spec.platform)
     end
@@ -549,6 +550,18 @@ def unlocking?
 
     private
 
+    def current_ruby_platform_locked?
+      return false unless generic_local_platform == Gem::Platform::RUBY
+
+      current_platform_locked?
+    end
+
+    def current_platform_locked?
+      @platforms.any? do |bundle_platform|
+        MatchPlatform.platforms_match?(bundle_platform, Bundler.local_platform)
+      end
+    end
+
     def add_current_platform
       add_platform(local_platform)
     end
@@ -871,8 +884,7 @@ def ruby_version_requirements(ruby_version)
       end
     end
 
-    def expand_dependencies(dependencies, remote = false, platforms = nil)
-      platforms ||= @platforms
+    def expand_dependencies(dependencies, remote = false)
       deps = []
       dependencies.each do |dep|
         dep = Dependency.new(dep, ">= 0") unless dep.respond_to?(:name)

bundler/lib/bundler/gem_helpers.rb

@@ -35,41 +35,33 @@ def local_platform
 
     def platform_specificity_match(spec_platform, user_platform)
       spec_platform = Gem::Platform.new(spec_platform)
-      return PlatformMatch::EXACT_MATCH if spec_platform == user_platform
-      return PlatformMatch::WORST_MATCH if spec_platform.nil? || spec_platform == Gem::Platform::RUBY || user_platform == Gem::Platform::RUBY
-
-      PlatformMatch.new(
-        PlatformMatch.os_match(spec_platform, user_platform),
-        PlatformMatch.cpu_match(spec_platform, user_platform),
-        PlatformMatch.platform_version_match(spec_platform, user_platform)
-      )
+
+      PlatformMatch.specificity_score(spec_platform, user_platform)
     end
     module_function :platform_specificity_match
 
     def select_best_platform_match(specs, platform)
-      specs.select {|spec| spec.match_platform(platform) }.
-        min_by {|spec| platform_specificity_match(spec.platform, platform) }
+      matching = specs.select {|spec| spec.match_platform(platform) }
+      exact = matching.select {|spec| spec.platform == platform }
+      return exact if exact.any?
+
+      sorted_matching = matching.sort_by {|spec| platform_specificity_match(spec.platform, platform) }
+      exemplary_spec = sorted_matching.first
+
+      sorted_matching.take_while{|spec| same_specificity(platform, spec, exemplary_spec) && same_deps(spec, exemplary_spec) }
     end
     module_function :select_best_platform_match
 
-    PlatformMatch = Struct.new(:os_match, :cpu_match, :platform_version_match)
     class PlatformMatch
-      def <=>(other)
-        return nil unless other.is_a?(PlatformMatch)
+      def self.specificity_score(spec_platform, user_platform)
+        return -1 if spec_platform == user_platform
+        return 1_000_000 if spec_platform.nil? || spec_platform == Gem::Platform::RUBY || user_platform == Gem::Platform::RUBY
 
-        m = os_match <=> other.os_match
-        return m unless m.zero?
-
-        m = cpu_match <=> other.cpu_match
-        return m unless m.zero?
-
-        m = platform_version_match <=> other.platform_version_match
-        m
+        os_match(spec_platform, user_platform) +
+          cpu_match(spec_platform, user_platform) * 10 +
+          platform_version_match(spec_platform, user_platform) * 100
       end
 
-      EXACT_MATCH = new(-1, -1, -1).freeze
-      WORST_MATCH = new(1_000_000, 1_000_000, 1_000_000).freeze
-
       def self.os_match(spec_platform, user_platform)
         if spec_platform.os == user_platform.os
           0
@@ -100,5 +92,19 @@ def self.platform_version_match(spec_platform, user_platform)
         end
       end
     end
+
+    def same_specificity(platform, spec, exemplary_spec)
+      platform_specificity_match(spec.platform, platform) == platform_specificity_match(exemplary_spec.platform, platform)
+    end
+    module_function :same_specificity
+
+    def same_deps(spec, exemplary_spec)
+      same_runtime_deps = spec.dependencies.sort == exemplary_spec.dependencies.sort
+      return same_runtime_deps unless spec.is_a?(Gem::Specification) && exemplary_spec.is_a?(Gem::Specification)
+
+      same_metadata_deps = spec.required_ruby_version == exemplary_spec.required_ruby_version && spec.required_rubygems_version == exemplary_spec.required_rubygems_version
+      same_runtime_deps && same_metadata_deps
+    end
+    module_function :same_deps
   end
 end

bundler/lib/bundler/lazy_specification.rb

@@ -4,7 +4,7 @@
 
 module Bundler
   class LazySpecification
-    Identifier = Struct.new(:name, :version, :source, :platform, :dependencies)
+    Identifier = Struct.new(:name, :version, :platform)
     class Identifier
       include Comparable
       def <=>(other)
@@ -108,7 +108,7 @@ def to_s
     end
 
     def identifier
-      @__identifier ||= Identifier.new(name, version, source, platform, dependencies)
+      @__identifier ||= Identifier.new(name, version, platform)
     end
 
     def git_version
@@ -131,17 +131,16 @@ def method_missing(method, *args, &blk)
     end
 
     #
-    # Bundler 2.2.0 was the first version that records the full resolution
-    # including platform specific gems in the lockfile, which means that if a
-    # gem with RUBY platform is recorded, the RUBY platform version of the gem
-    # should be installed. Previously bundler would record only generic versions
-    # in the lockfile and then install the most specific platform variant if
-    # available.
+    # For backwards compatibility with existing lockfiles, if the most specific
+    # locked platform is RUBY, we keep the previous behaviour of resolving the
+    # best platform variant at materiliazation time. For previous bundler
+    # versions (before 2.2.0) this was always the case (except when the lockfile
+    # only included non-ruby platforms), but we're also keeping this behaviour
+    # on newer bundlers unless users generate the lockfile from scratch or
+    # explicitly add a more specific platform.
     #
     def ruby_platform_materializes_to_ruby_platform?
-      locked_bundler_version = Bundler.locked_bundler_version
-
-      locked_bundler_version.nil? || Gem::Version.new(locked_bundler_version) >= Gem::Version.new("2.2.0")
+      !Bundler.most_specific_locked_platform?(Gem::Platform::RUBY)
     end
   end
 end