From 210219119980e56440b9bc1c6245304062987804 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 26 May 2026 16:47:54 +0900 Subject: [PATCH 1/6] Allow Bundler::LockfileParser to accept an explicit lockfile_path The path was previously inferred from Bundler.default_lockfile via SharedHelpers, which couples the parser to a configured Bundler environment. Accepting lockfile_path: as a keyword argument lets external callers (e.g. Gem::RequestSet) instantiate the parser without setting up Bundler globals, while preserving the previous fallback. --- bundler/lib/bundler/lockfile_parser.rb | 4 ++-- spec/bundler/lockfile_parser_spec.rb | 21 +++++++++++++++++++++ spec/quality_spec.rb | 2 +- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/bundler/lib/bundler/lockfile_parser.rb b/bundler/lib/bundler/lockfile_parser.rb index 9e21f8f1d2a1..852fc631f3b1 100644 --- a/bundler/lib/bundler/lockfile_parser.rb +++ b/bundler/lib/bundler/lockfile_parser.rb @@ -95,14 +95,14 @@ def self.bundled_with lockfile_contents.split(BUNDLED).last.strip end - def initialize(lockfile, strict: false) + def initialize(lockfile, strict: false, lockfile_path: nil) @platforms = [] @sources = [] @metadata_source = Source::Metadata.new @dependencies = {} @parse_method = nil @specs = {} - @lockfile_path = begin + @lockfile_path = lockfile_path || begin SharedHelpers.relative_lockfile_path rescue GemfileNotFound "Gemfile.lock" diff --git a/spec/bundler/lockfile_parser_spec.rb b/spec/bundler/lockfile_parser_spec.rb index 4b493a37576b..7364ab98e541 100644 --- a/spec/bundler/lockfile_parser_spec.rb +++ b/spec/bundler/lockfile_parser_spec.rb @@ -252,6 +252,27 @@ end end + context "when lockfile_path is given" do + it "uses the provided path in error messages instead of looking up Bundler.default_lockfile" do + expect(Bundler::SharedHelpers).not_to receive(:relative_lockfile_path) + parser = described_class.new(lockfile_contents, lockfile_path: "custom/path.lock") + expect(parser.valid?).to be(true) + rake_spec = parser.specs.last + checksums = parser.sources.last.checksum_store.to_lock(rake_spec) + expected_checksum = Bundler::Checksum.from_lock( + "sha256=814828c34f1315d7e7b7e8295184577cc4e969bad6156ac069d02d63f58d82e8", + "custom/path.lock:20:17" + ) + expect(checksums).to eq("#{rake_spec.lock_name} #{expected_checksum.to_lock}") + end + + it "raises with the provided path when the lockfile contains merge conflicts" do + expect do + described_class.new("<<<<<<<\n", lockfile_path: "custom/path.lock") + end.to raise_error(Bundler::LockfileError, %r{custom/path\.lock contains merge conflicts}) + end + end + context "when CHECKSUMS has duplicate checksums in the lockfile that don't match" do let(:bad_checksum) { "sha256=c0ffee11c0ffee11c0ffee11c0ffee11c0ffee11c0ffee11c0ffee11c0ffee11" } let(:lockfile_contents) { super().split(/(?<=CHECKSUMS\n)/m).insert(1, " rake (10.3.2) #{bad_checksum}\n").join } diff --git a/spec/quality_spec.rb b/spec/quality_spec.rb index 33ae503a5a3e..16b7f1878807 100644 --- a/spec/quality_spec.rb +++ b/spec/quality_spec.rb @@ -109,7 +109,7 @@ def check_for_specific_pronouns(filename) it "does not include any unresolved merge conflicts" do error_messages = [] - exempt = %r{lock/lockfile_spec|quality_spec|vcr_cassettes|\.ronn|lockfile_parser\.rb} + exempt = %r{lock/lockfile_spec|quality_spec|vcr_cassettes|\.ronn|lockfile_parser} tracked_files.each do |filename| next if filename&.match?(exempt) error_messages << check_for_git_merge_conflicts(filename) From af99bdf78ddcbe5cbe13cb358dfc18c2649c73da Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 26 May 2026 16:50:24 +0900 Subject: [PATCH 2/6] Add load_gemdeps lockfile section coverage to test_gem_request_set These tests pin down the observable RequestSet state produced when load_gemdeps reads a lockfile with GEM, GIT, or PATH sections, so the upcoming switch to Bundler::LockfileParser can be evaluated against a clear behavioral baseline. --- test/rubygems/test_gem_request_set.rb | 104 ++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/test/rubygems/test_gem_request_set.rb b/test/rubygems/test_gem_request_set.rb index 9aa244892c69..6ebc95ea20f2 100644 --- a/test/rubygems/test_gem_request_set.rb +++ b/test/rubygems/test_gem_request_set.rb @@ -311,6 +311,110 @@ def test_load_gemdeps_without_groups assert_empty rs.dependencies end + def test_load_gemdeps_with_lockfile_gem_section + rs = Gem::RequestSet.new + + File.open "gem.deps.rb", "w" do |io| + io.puts 'gem "b"' + end + + File.open "gem.deps.rb.lock", "w" do |io| + io.puts <<~LOCKFILE + GEM + remote: #{@gem_repo} + specs: + a (1) + b (1) + a (~> 1.0) + + PLATFORMS + #{Gem::Platform::RUBY} + + DEPENDENCIES + b + LOCKFILE + end + + rs.load_gemdeps "gem.deps.rb" + + lock_set = rs.sets.find {|set| Gem::Resolver::LockSet === set } + refute_nil lock_set, "LockSet should be created from GEM section" + assert_equal %w[a-1 b-1], lock_set.specs.map(&:full_name).sort + end + + def test_load_gemdeps_with_lockfile_git_section + rs = Gem::RequestSet.new + + File.open "gem.deps.rb", "w" do |io| + io.puts 'gem "a", :git => "git://example/a.git"' + end + + File.open "gem.deps.rb.lock", "w" do |io| + io.puts <<~LOCKFILE + GIT + remote: git://example/a.git + revision: deadbeef + specs: + a (1) + + PLATFORMS + #{Gem::Platform::RUBY} + + DEPENDENCIES + a! + LOCKFILE + end + + rs.load_gemdeps "gem.deps.rb" + + git_set = rs.sets.find {|set| Gem::Resolver::GitSet === set } + refute_nil git_set, "GitSet should be created from GIT section" + assert_includes git_set.specs.keys, "a" + end + + def test_load_gemdeps_with_lockfile_path_section + _, _, directory = vendor_gem + + rs = Gem::RequestSet.new + + File.open "gem.deps.rb", "w" do |io| + io.puts "gem \"a\", :path => #{directory.inspect}" + end + + File.open "gem.deps.rb.lock", "w" do |io| + io.puts <<~LOCKFILE + PATH + remote: #{directory} + specs: + a (1) + + PLATFORMS + #{Gem::Platform::RUBY} + + DEPENDENCIES + a! + LOCKFILE + end + + rs.load_gemdeps "gem.deps.rb" + + vendor_set = rs.sets.find {|set| Gem::Resolver::VendorSet === set } + refute_nil vendor_set, "VendorSet should be created from PATH section" + assert_equal %w[a-1], vendor_set.specs.values.map(&:full_name) + end + + def test_load_gemdeps_with_missing_lockfile + rs = Gem::RequestSet.new + + File.open "gem.deps.rb", "w" do |io| + io.puts 'gem "a"' + end + + rs.load_gemdeps "gem.deps.rb" + + assert_equal [dep("a")], rs.dependencies + end + def test_resolve a = util_spec "a", "2", "b" => ">= 2" b = util_spec "b", "2" From 7aeba14feb1d4e4b56f319533d95e16a83cafc8c Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 26 May 2026 16:56:31 +0900 Subject: [PATCH 3/6] Use Bundler::LockfileParser inside Gem::RequestSet#load_gemdeps The rubygems-specific tokenizer/parser pair is replaced by Bundler's LockfileParser, with a small adapter that translates the parsed LazySpecifications and Bundler sources back into the LockSet, GitSet, and VendorSet objects that the resolver expects. This unifies lockfile parsing on a single implementation so the standalone tokenizer/parser sources and tests can be retired in subsequent commits. --- Rakefile | 1 + lib/rubygems/request_set.rb | 64 ++++++++++++++++++++++++++++++++++--- 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/Rakefile b/Rakefile index 7f1b894c9a6a..0b91b23aaed3 100644 --- a/Rakefile +++ b/Rakefile @@ -74,6 +74,7 @@ Rake::TestTask.new do |t| t.ruby_opts.unshift("--disable-gems", "-r#{coverage_setup}") end + t.libs << "bundler/lib" t.libs << "test" t.test_files = FileList["test/**/test_*.rb"] end diff --git a/lib/rubygems/request_set.rb b/lib/rubygems/request_set.rb index dbebd1af0c37..a9b38f2e008c 100644 --- a/lib/rubygems/request_set.rb +++ b/lib/rubygems/request_set.rb @@ -322,11 +322,8 @@ def load_gemdeps(path, without_groups = [], installing = false) @git_set.root_dir = @install_dir lock_file = "#{File.expand_path(path)}.lock" - begin - tokenizer = Gem::RequestSet::Lockfile::Tokenizer.from_file lock_file - parser = tokenizer.make_parser self, [] - parser.parse - rescue Errno::ENOENT + if File.exist?(lock_file) + load_lockfile lock_file end gf = Gem::RequestSet::GemDependencyAPI.new self, path @@ -335,6 +332,63 @@ def load_gemdeps(path, without_groups = [], installing = false) gf.load end + def load_lockfile(lock_file) # :nodoc: + require "bundler" + require "bundler/lockfile_parser" + + # Bundler::Source::Path resolves relative `remote:` paths against + # Bundler.root, which raises when there is no Gemfile in the working + # directory. Anchor it to the lockfile's directory so PATH sections in a + # `gem install -g` lockfile can be parsed without a Bundler environment. + previous_root = Bundler.instance_variable_get(:@root) + Bundler.instance_variable_set(:@root, Pathname.new(File.expand_path(File.dirname(lock_file)))) + + parser = Bundler::LockfileParser.new(File.read(lock_file), lockfile_path: lock_file) + + parser.specs.group_by(&:source).each do |source, specs| + case source + when Bundler::Source::Rubygems + remotes = source.remotes.map {|remote| Gem::Source.new(remote.to_s) } + remotes << Gem::Source.new(Gem::DEFAULT_HOST) if remotes.empty? + lock_set = Gem::Resolver::LockSet.new(remotes) + specs.each do |spec| + added = lock_set.add(spec.name, spec.version.to_s, spec.platform) + spec.dependencies.each do |dep| + added.each {|s| s.add_dependency dep } + end + end + @sets << lock_set + when Bundler::Source::Git + git_set = Gem::Resolver::GitSet.new + git_set.root_dir = @install_dir + specs.each do |spec| + git_spec = git_set.add_git_spec( + spec.name, + spec.version.to_s, + source.uri.to_s, + source.revision, + source.submodules || false + ) + spec.dependencies.each {|dep| git_spec.add_dependency dep } + end + @sets << git_set + when Bundler::Source::Path + vendor_set = Gem::Resolver::VendorSet.new + specs.each do |spec| + loaded = vendor_set.add_vendor_gem(spec.name, source.path.to_s) + spec.dependencies.each {|dep| loaded.dependencies << dep } + end + @sets << vendor_set + end + end + + parser.dependencies.each_value do |dep| + gem dep.name, *dep.requirement.as_list + end + ensure + Bundler.instance_variable_set(:@root, previous_root) if defined?(previous_root) + end + def pretty_print(q) # :nodoc: q.group 2, "[RequestSet:", "]" do q.breakable From 5cfef5efa6fa54e01e3b9a4f7fdb609c1316cc4a Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 26 May 2026 16:57:07 +0900 Subject: [PATCH 4/6] Remove unit tests for the retired lockfile tokenizer and parser Both files exercised Gem::RequestSet::Lockfile::Tokenizer and ::Parser directly, which are no longer used by load_gemdeps now that Bundler::LockfileParser handles the read path. End-to-end coverage of the GEM, GIT, and PATH sections lives in test_gem_request_set. --- .../test_gem_request_set_lockfile_parser.rb | 544 ------------------ ...test_gem_request_set_lockfile_tokenizer.rb | 307 ---------- 2 files changed, 851 deletions(-) delete mode 100644 test/rubygems/test_gem_request_set_lockfile_parser.rb delete mode 100644 test/rubygems/test_gem_request_set_lockfile_tokenizer.rb diff --git a/test/rubygems/test_gem_request_set_lockfile_parser.rb b/test/rubygems/test_gem_request_set_lockfile_parser.rb deleted file mode 100644 index 253a59b243cb..000000000000 --- a/test/rubygems/test_gem_request_set_lockfile_parser.rb +++ /dev/null @@ -1,544 +0,0 @@ -# frozen_string_literal: true - -require_relative "helper" -require "rubygems/request_set" -require "rubygems/request_set/lockfile" -require "rubygems/request_set/lockfile/tokenizer" -require "rubygems/request_set/lockfile/parser" - -class TestGemRequestSetLockfileParser < Gem::TestCase - def setup - super - @gem_deps_file = "gem.deps.rb" - @lock_file = File.expand_path "#{@gem_deps_file}.lock" - @set = Gem::RequestSet.new - end - - def test_get - tokenizer = Gem::RequestSet::Lockfile::Tokenizer.new "\n" - parser = tokenizer.make_parser nil, nil - - assert_equal :newline, parser.get.first - end - - def test_get_type_mismatch - filename = File.expand_path("#{@gem_deps_file}.lock") - tokenizer = Gem::RequestSet::Lockfile::Tokenizer.new "foo", filename, 1, 0 - parser = tokenizer.make_parser nil, nil - - e = assert_raise Gem::RequestSet::Lockfile::ParseError do - parser.get :section - end - - expected = - 'unexpected token [:text, "foo"], expected :section (at line 1 column 0)' - - assert_equal expected, e.message - - assert_equal 1, e.line - assert_equal 0, e.column - assert_equal filename, e.path - end - - def test_get_type_multiple - filename = File.expand_path("#{@gem_deps_file}.lock") - tokenizer = Gem::RequestSet::Lockfile::Tokenizer.new "x", filename, 1 - parser = tokenizer.make_parser nil, nil - - assert parser.get [:text, :section] - end - - def test_get_type_value_mismatch - filename = File.expand_path("#{@gem_deps_file}.lock") - tokenizer = Gem::RequestSet::Lockfile::Tokenizer.new "x", filename, 1 - parser = tokenizer.make_parser nil, nil - - e = assert_raise Gem::RequestSet::Lockfile::ParseError do - parser.get :text, "y" - end - - expected = - 'unexpected token [:text, "x"], expected [:text, "y"] (at line 1 column 0)' - - assert_equal expected, e.message - - assert_equal 1, e.line - assert_equal 0, e.column - assert_equal File.expand_path("#{@gem_deps_file}.lock"), e.path - end - - def test_parse - write_lockfile <<-LOCKFILE.strip -GEM - remote: #{@gem_repo} - specs: - a (2) - -PLATFORMS - #{Gem::Platform::RUBY} - -DEPENDENCIES - a - LOCKFILE - - platforms = [] - parse_lockfile @set, platforms - - assert_equal [dep("a")], @set.dependencies - - assert_equal [Gem::Platform::RUBY], platforms - - lockfile_set = @set.sets.find do |set| - Gem::Resolver::LockSet === set - end - - assert lockfile_set, "could not find a LockSet" - - assert_equal %w[a-2], lockfile_set.specs.map(&:full_name) - end - - def test_parse_dependencies - write_lockfile <<-LOCKFILE -GEM - remote: #{@gem_repo} - specs: - a (2) - -PLATFORMS - #{Gem::Platform::RUBY} - -DEPENDENCIES - a (>= 1, <= 2) - LOCKFILE - - platforms = [] - parse_lockfile @set, platforms - - assert_equal [dep("a", ">= 1", "<= 2")], @set.dependencies - - assert_equal [Gem::Platform::RUBY], platforms - - lockfile_set = @set.sets.find do |set| - Gem::Resolver::LockSet === set - end - - assert lockfile_set, "could not find a LockSet" - - assert_equal %w[a-2], lockfile_set.specs.map(&:full_name) - end - - def test_parse_DEPENDENCIES_git - write_lockfile <<-LOCKFILE -GIT - remote: git://git.example/josevalim/rails-footnotes.git - revision: 3a6ac1971e91d822f057650cc5916ebfcbd6ee37 - specs: - rails-footnotes (3.7.9) - rails (>= 3.0.0) - -GIT - remote: git://git.example/svenfuchs/i18n-active_record.git - revision: 55507cf59f8f2173d38e07e18df0e90d25b1f0f6 - specs: - i18n-active_record (0.0.2) - i18n (>= 0.5.0) - -GEM - remote: http://gems.example/ - specs: - i18n (0.6.9) - rails (4.0.0) - -PLATFORMS - ruby - -DEPENDENCIES - i18n-active_record! - rails-footnotes! - LOCKFILE - - parse_lockfile @set, [] - - expected = [ - dep("i18n-active_record", "= 0.0.2"), - dep("rails-footnotes", "= 3.7.9"), - ] - - assert_equal expected, @set.dependencies - end - - def test_parse_DEPENDENCIES_git_version - write_lockfile <<-LOCKFILE -GIT - remote: git://github.com/progrium/ruby-jwt.git - revision: 8d74770c6cd92ea234b428b5d0c1f18306a4f41c - specs: - jwt (1.1) - -GEM - remote: http://gems.example/ - specs: - -PLATFORMS - ruby - -DEPENDENCIES - jwt (= 1.1)! - LOCKFILE - - parse_lockfile @set, [] - - expected = [ - dep("jwt", "= 1.1"), - ] - - assert_equal expected, @set.dependencies - end - - def test_parse_GEM - write_lockfile <<-LOCKFILE -GEM - specs: - a (2) - -PLATFORMS - ruby - -DEPENDENCIES - a - LOCKFILE - - parse_lockfile @set, [] - - assert_equal [dep("a", ">= 0")], @set.dependencies - - lockfile_set = @set.sets.find do |set| - Gem::Resolver::LockSet === set - end - - assert lockfile_set, "found a LockSet" - - assert_equal %w[a-2], lockfile_set.specs.map(&:full_name) - end - - def test_parse_GEM_remote_multiple - write_lockfile <<-LOCKFILE -GEM - remote: https://gems.example/ - remote: https://other.example/ - specs: - a (2) - -PLATFORMS - ruby - -DEPENDENCIES - a - LOCKFILE - - parse_lockfile @set, [] - - assert_equal [dep("a", ">= 0")], @set.dependencies - - lockfile_set = @set.sets.find do |set| - Gem::Resolver::LockSet === set - end - - assert lockfile_set, "found a LockSet" - - assert_equal %w[a-2], lockfile_set.specs.map(&:full_name) - - assert_equal %w[https://gems.example/ https://other.example/], - lockfile_set.specs.flat_map {|s| s.sources.map {|src| src.uri.to_s } } - end - - def test_parse_GIT - @set.instance_variable_set :@install_dir, "install_dir" - - write_lockfile <<-LOCKFILE -GIT - remote: git://example/a.git - revision: abranch - specs: - a (2) - b (>= 3) - c - -DEPENDENCIES - a! - LOCKFILE - - parse_lockfile @set, [] - - assert_equal [dep("a", "= 2")], @set.dependencies - - lockfile_set = @set.sets.find do |set| - Gem::Resolver::LockSet === set - end - - refute lockfile_set, "found a LockSet" - - git_set = @set.sets.find do |set| - Gem::Resolver::GitSet === set - end - - assert git_set, "could not find a GitSet" - - assert_equal %w[a-2], git_set.specs.values.map(&:full_name) - - assert_equal [dep("b", ">= 3"), dep("c")], - git_set.specs.values.first.dependencies - - expected = { - "a" => %w[git://example/a.git abranch], - } - - assert_equal expected, git_set.repositories - assert_equal "install_dir", git_set.root_dir - end - - def test_parse_GIT_branch - write_lockfile <<-LOCKFILE -GIT - remote: git://example/a.git - revision: 1234abc - branch: 0-9-12-stable - specs: - a (2) - b (>= 3) - -DEPENDENCIES - a! - LOCKFILE - - parse_lockfile @set, [] - - assert_equal [dep("a", "= 2")], @set.dependencies - - lockfile_set = @set.sets.find do |set| - Gem::Resolver::LockSet === set - end - - refute lockfile_set, "found a LockSet" - - git_set = @set.sets.find do |set| - Gem::Resolver::GitSet === set - end - - assert git_set, "could not find a GitSet" - - expected = { - "a" => %w[git://example/a.git 1234abc], - } - - assert_equal expected, git_set.repositories - end - - def test_parse_GIT_ref - write_lockfile <<-LOCKFILE -GIT - remote: git://example/a.git - revision: 1234abc - ref: 1234abc - specs: - a (2) - b (>= 3) - -DEPENDENCIES - a! - LOCKFILE - - parse_lockfile @set, [] - - assert_equal [dep("a", "= 2")], @set.dependencies - - lockfile_set = @set.sets.find do |set| - Gem::Resolver::LockSet === set - end - - refute lockfile_set, "found a LockSet" - - git_set = @set.sets.find do |set| - Gem::Resolver::GitSet === set - end - - assert git_set, "could not find a GitSet" - - expected = { - "a" => %w[git://example/a.git 1234abc], - } - - assert_equal expected, git_set.repositories - end - - def test_parse_GIT_tag - write_lockfile <<-LOCKFILE -GIT - remote: git://example/a.git - revision: 1234abc - tag: v0.9.12 - specs: - a (2) - b (>= 3) - -DEPENDENCIES - a! - LOCKFILE - - parse_lockfile @set, [] - - assert_equal [dep("a", "= 2")], @set.dependencies - - lockfile_set = @set.sets.find do |set| - Gem::Resolver::LockSet === set - end - - refute lockfile_set, "found a LockSet" - - git_set = @set.sets.find do |set| - Gem::Resolver::GitSet === set - end - - assert git_set, "could not find a GitSet" - - expected = { - "a" => %w[git://example/a.git 1234abc], - } - - assert_equal expected, git_set.repositories - end - - def test_parse_PATH - _, _, directory = vendor_gem - - write_lockfile <<-LOCKFILE -PATH - remote: #{directory} - specs: - a (1) - b (2) - -DEPENDENCIES - a! - LOCKFILE - - parse_lockfile @set, [] - - assert_equal [dep("a", "= 1")], @set.dependencies - - lockfile_set = @set.sets.find do |set| - Gem::Resolver::LockSet === set - end - - refute lockfile_set, "found a LockSet" - - vendor_set = @set.sets.find do |set| - Gem::Resolver::VendorSet === set - end - - assert vendor_set, "could not find a VendorSet" - - assert_equal %w[a-1], vendor_set.specs.values.map(&:full_name) - - spec = vendor_set.load_spec "a", nil, nil, nil - - assert_equal [dep("b", "= 2")], spec.dependencies - end - - def test_parse_dependency - write_lockfile " 1)" - - tokenizer = Gem::RequestSet::Lockfile::Tokenizer.from_file @lock_file - parser = tokenizer.make_parser nil, nil - - parsed = parser.parse_dependency "a", "=" - - assert_equal dep("a", "= 1"), parsed - - write_lockfile ")" - - tokenizer = Gem::RequestSet::Lockfile::Tokenizer.from_file @lock_file - parser = tokenizer.make_parser nil, nil - - parsed = parser.parse_dependency "a", "2" - - assert_equal dep("a", "= 2"), parsed - end - - def test_parse_gem_specs_dependency - write_lockfile <<-LOCKFILE -GEM - remote: #{@gem_repo} - specs: - a (2) - b (= 3) - c (~> 4) - d - e (~> 5.0, >= 5.0.1) - b (3-x86_64-linux) - -PLATFORMS - #{Gem::Platform::RUBY} - -DEPENDENCIES - a - LOCKFILE - - platforms = [] - parse_lockfile @set, platforms - - assert_equal [dep("a")], @set.dependencies - - assert_equal [Gem::Platform::RUBY], platforms - - lockfile_set = @set.sets.find do |set| - Gem::Resolver::LockSet === set - end - - assert lockfile_set, "could not find a LockSet" - - assert_equal %w[a-2 b-3], lockfile_set.specs.map(&:full_name) - - expected = [ - Gem::Platform::RUBY, - Gem::Platform.new("x86_64-linux"), - ] - - assert_equal expected, lockfile_set.specs.map(&:platform) - - spec = lockfile_set.specs.first - - expected = [ - dep("b", "= 3"), - dep("c", "~> 4"), - dep("d"), - dep("e", "~> 5.0", ">= 5.0.1"), - ] - - assert_equal expected, spec.dependencies - end - - def test_parse_missing - assert_raise(Errno::ENOENT) do - parse_lockfile @set, [] - end - - lockfile_set = @set.sets.find do |set| - Gem::Resolver::LockSet === set - end - - refute lockfile_set - end - - def write_lockfile(lockfile) - File.open @lock_file, "w" do |io| - io.write lockfile - end - end - - def parse_lockfile(set, platforms) - tokenizer = Gem::RequestSet::Lockfile::Tokenizer.from_file @lock_file - parser = tokenizer.make_parser set, platforms - parser.parse - end -end diff --git a/test/rubygems/test_gem_request_set_lockfile_tokenizer.rb b/test/rubygems/test_gem_request_set_lockfile_tokenizer.rb deleted file mode 100644 index dce8c9ada5a6..000000000000 --- a/test/rubygems/test_gem_request_set_lockfile_tokenizer.rb +++ /dev/null @@ -1,307 +0,0 @@ -# frozen_string_literal: true - -require_relative "helper" -require "rubygems/request_set" -require "rubygems/request_set/lockfile" -require "rubygems/request_set/lockfile/tokenizer" -require "rubygems/request_set/lockfile/parser" - -class TestGemRequestSetLockfileTokenizer < Gem::TestCase - def setup - super - - @gem_deps_file = "gem.deps.rb" - @lock_file = File.expand_path "#{@gem_deps_file}.lock" - end - - def test_peek - tokenizer = Gem::RequestSet::Lockfile::Tokenizer.new "\n" - - assert_equal :newline, tokenizer.peek.first - - assert_equal :newline, tokenizer.next_token.first - - assert_equal :EOF, tokenizer.peek.first - end - - def test_skip - tokenizer = Gem::RequestSet::Lockfile::Tokenizer.new "\n" - - refute_predicate tokenizer, :empty? - - tokenizer.skip :newline - - assert_empty tokenizer - end - - def test_token_pos - tokenizer = Gem::RequestSet::Lockfile::Tokenizer.new "" - assert_equal [5, 0], tokenizer.token_pos(5) - - tokenizer = Gem::RequestSet::Lockfile::Tokenizer.new "", nil, 1, 2 - assert_equal [3, 1], tokenizer.token_pos(5) - end - - def test_tokenize - write_lockfile <<-LOCKFILE -GEM - remote: #{@gem_repo} - specs: - a (2) - b (= 2) - c (!= 3) - d (> 4) - e (< 5) - f (>= 6) - g (<= 7) - h (~> 8) - -PLATFORMS - #{Gem::Platform::RUBY} - -DEPENDENCIES - a - LOCKFILE - - expected = [ - [:section, "GEM", 0, 0], - [:newline, nil, 3, 0], - - [:entry, "remote", 2, 1], - [:text, @gem_repo, 10, 1], - [:newline, nil, 34, 1], - - [:entry, "specs", 2, 2], - [:newline, nil, 8, 2], - - [:text, "a", 4, 3], - [:l_paren, nil, 6, 3], - [:text, "2", 7, 3], - [:r_paren, nil, 8, 3], - [:newline, nil, 9, 3], - - [:text, "b", 6, 4], - [:l_paren, nil, 8, 4], - [:requirement, "=", 9, 4], - [:text, "2", 11, 4], - [:r_paren, nil, 12, 4], - [:newline, nil, 13, 4], - - [:text, "c", 6, 5], - [:l_paren, nil, 8, 5], - [:requirement, "!=", 9, 5], - [:text, "3", 12, 5], - [:r_paren, nil, 13, 5], - [:newline, nil, 14, 5], - - [:text, "d", 6, 6], - [:l_paren, nil, 8, 6], - [:requirement, ">", 9, 6], - [:text, "4", 11, 6], - [:r_paren, nil, 12, 6], - [:newline, nil, 13, 6], - - [:text, "e", 6, 7], - [:l_paren, nil, 8, 7], - [:requirement, "<", 9, 7], - [:text, "5", 11, 7], - [:r_paren, nil, 12, 7], - [:newline, nil, 13, 7], - - [:text, "f", 6, 8], - [:l_paren, nil, 8, 8], - [:requirement, ">=", 9, 8], - [:text, "6", 12, 8], - [:r_paren, nil, 13, 8], - [:newline, nil, 14, 8], - - [:text, "g", 6, 9], - [:l_paren, nil, 8, 9], - [:requirement, "<=", 9, 9], - [:text, "7", 12, 9], - [:r_paren, nil, 13, 9], - [:newline, nil, 14, 9], - - [:text, "h", 6, 10], - [:l_paren, nil, 8, 10], - [:requirement, "~>", 9, 10], - [:text, "8", 12, 10], - [:r_paren, nil, 13, 10], - [:newline, nil, 14, 10], - - [:newline, nil, 0, 11], - - [:section, "PLATFORMS", 0, 12], - [:newline, nil, 9, 12], - - [:text, Gem::Platform::RUBY, 2, 13], - [:newline, nil, 6, 13], - - [:newline, nil, 0, 14], - - [:section, "DEPENDENCIES", 0, 15], - [:newline, nil, 12, 15], - - [:text, "a", 2, 16], - [:newline, nil, 3, 16], - ] - - assert_equal expected, tokenize_lockfile - end - - def test_tokenize_capitals - write_lockfile <<-LOCKFILE -GEM - remote: #{@gem_repo} - specs: - Ab (2) - -PLATFORMS - #{Gem::Platform::RUBY} - -DEPENDENCIES - Ab - LOCKFILE - - expected = [ - [:section, "GEM", 0, 0], - [:newline, nil, 3, 0], - [:entry, "remote", 2, 1], - [:text, @gem_repo, 10, 1], - [:newline, nil, 34, 1], - [:entry, "specs", 2, 2], - [:newline, nil, 8, 2], - [:text, "Ab", 4, 3], - [:l_paren, nil, 7, 3], - [:text, "2", 8, 3], - [:r_paren, nil, 9, 3], - [:newline, nil, 10, 3], - [:newline, nil, 0, 4], - [:section, "PLATFORMS", 0, 5], - [:newline, nil, 9, 5], - [:text, Gem::Platform::RUBY, 2, 6], - [:newline, nil, 6, 6], - [:newline, nil, 0, 7], - [:section, "DEPENDENCIES", 0, 8], - [:newline, nil, 12, 8], - [:text, "Ab", 2, 9], - [:newline, nil, 4, 9], - ] - - assert_equal expected, tokenize_lockfile - end - - def test_tokenize_conflict_markers - write_lockfile "<<<<<<<" - - e = assert_raise Gem::RequestSet::Lockfile::ParseError do - tokenize_lockfile - end - - assert_equal "your #{@lock_file} contains merge conflict markers (at line 0 column 0)", - e.message - - write_lockfile "|||||||" - - e = assert_raise Gem::RequestSet::Lockfile::ParseError do - tokenize_lockfile - end - - assert_equal "your #{@lock_file} contains merge conflict markers (at line 0 column 0)", - e.message - - write_lockfile "=======" - - e = assert_raise Gem::RequestSet::Lockfile::ParseError do - tokenize_lockfile - end - - assert_equal "your #{@lock_file} contains merge conflict markers (at line 0 column 0)", - e.message - - write_lockfile ">>>>>>>" - - e = assert_raise Gem::RequestSet::Lockfile::ParseError do - tokenize_lockfile - end - - assert_equal "your #{@lock_file} contains merge conflict markers (at line 0 column 0)", - e.message - end - - def test_tokenize_git - write_lockfile <<-LOCKFILE -DEPENDENCIES - a! - LOCKFILE - - expected = [ - [:section, "DEPENDENCIES", 0, 0], - [:newline, nil, 12, 0], - - [:text, "a", 2, 1], - [:bang, nil, 3, 1], - [:newline, nil, 4, 1], - ] - - assert_equal expected, tokenize_lockfile - end - - def test_tokenize_multiple - write_lockfile <<-LOCKFILE -GEM - remote: #{@gem_repo} - specs: - a (2) - b (~> 3.0, >= 3.0.1) - LOCKFILE - - expected = [ - [:section, "GEM", 0, 0], - [:newline, nil, 3, 0], - - [:entry, "remote", 2, 1], - [:text, @gem_repo, 10, 1], - [:newline, nil, 34, 1], - - [:entry, "specs", 2, 2], - [:newline, nil, 8, 2], - - [:text, "a", 4, 3], - [:l_paren, nil, 6, 3], - [:text, "2", 7, 3], - [:r_paren, nil, 8, 3], - [:newline, nil, 9, 3], - - [:text, "b", 6, 4], - [:l_paren, nil, 8, 4], - [:requirement, "~>", 9, 4], - [:text, "3.0", 12, 4], - [:comma, nil, 15, 4], - [:requirement, ">=", 17, 4], - [:text, "3.0.1", 20, 4], - [:r_paren, nil, 25, 4], - [:newline, nil, 26, 4], - ] - - assert_equal expected, tokenize_lockfile - end - - def test_unget - tokenizer = Gem::RequestSet::Lockfile::Tokenizer.new "\n" - tokenizer.unshift :token - parser = tokenizer.make_parser nil, nil - - assert_equal :token, parser.get - end - - def write_lockfile(lockfile) - File.open @lock_file, "w" do |io| - io.write lockfile - end - end - - def tokenize_lockfile - Gem::RequestSet::Lockfile::Tokenizer.from_file(@lock_file).to_a - end -end From e757cd59780404ca74743671e5e4100078c3ac0f Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 26 May 2026 16:58:28 +0900 Subject: [PATCH 5/6] Drop the rubygems-specific lockfile tokenizer and parser sources These files implemented the read path that Bundler::LockfileParser now covers via the adapter in Gem::RequestSet#load_lockfile. The Lockfile class itself is kept because Gem::RequestSet#install_from_gemdeps still uses Lockfile.build / Lockfile#write for emitting .lock files, which will be migrated separately. --- lib/rubygems/request_set.rb | 1 - lib/rubygems/request_set/lockfile.rb | 2 - lib/rubygems/request_set/lockfile/parser.rb | 344 ------------------ .../request_set/lockfile/tokenizer.rb | 122 ------- 4 files changed, 469 deletions(-) delete mode 100644 lib/rubygems/request_set/lockfile/parser.rb delete mode 100644 lib/rubygems/request_set/lockfile/tokenizer.rb diff --git a/lib/rubygems/request_set.rb b/lib/rubygems/request_set.rb index a9b38f2e008c..c06ef32da949 100644 --- a/lib/rubygems/request_set.rb +++ b/lib/rubygems/request_set.rb @@ -516,4 +516,3 @@ def tsort_each_child(node) # :nodoc: require_relative "request_set/gem_dependency_api" require_relative "request_set/lockfile" -require_relative "request_set/lockfile/tokenizer" diff --git a/lib/rubygems/request_set/lockfile.rb b/lib/rubygems/request_set/lockfile.rb index da6dd329bc07..8b9c9690d622 100644 --- a/lib/rubygems/request_set/lockfile.rb +++ b/lib/rubygems/request_set/lockfile.rb @@ -231,5 +231,3 @@ def requests @set.sorted_requests end end - -require_relative "lockfile/tokenizer" diff --git a/lib/rubygems/request_set/lockfile/parser.rb b/lib/rubygems/request_set/lockfile/parser.rb deleted file mode 100644 index e751a1445e31..000000000000 --- a/lib/rubygems/request_set/lockfile/parser.rb +++ /dev/null @@ -1,344 +0,0 @@ -# frozen_string_literal: true - -class Gem::RequestSet::Lockfile::Parser - ### - # Parses lockfiles - - def initialize(tokenizer, set, platforms, filename = nil) - @tokens = tokenizer - @filename = filename - @set = set - @platforms = platforms - end - - def parse - until @tokens.empty? do - token = get - - case token.type - when :section then - @tokens.skip :newline - - case token.value - when "DEPENDENCIES" then - parse_DEPENDENCIES - when "GIT" then - parse_GIT - when "GEM" then - parse_GEM - when "PATH" then - parse_PATH - when "PLATFORMS" then - parse_PLATFORMS - else - token = get until @tokens.empty? || peek.first == :section - end - else - raise "BUG: unhandled token #{token.type} (#{token.value.inspect}) at line #{token.line} column #{token.column}" - end - end - end - - ## - # Gets the next token for a Lockfile - - def get(expected_types = nil, expected_value = nil) # :nodoc: - token = @tokens.shift - - if expected_types && !Array(expected_types).include?(token.type) - unget token - - message = "unexpected token [#{token.type.inspect}, #{token.value.inspect}], " \ - "expected #{expected_types.inspect}" - - raise Gem::RequestSet::Lockfile::ParseError.new message, token.column, token.line, @filename - end - - if expected_value && expected_value != token.value - unget token - - message = "unexpected token [#{token.type.inspect}, #{token.value.inspect}], " \ - "expected [#{expected_types.inspect}, " \ - "#{expected_value.inspect}]" - - raise Gem::RequestSet::Lockfile::ParseError.new message, token.column, token.line, @filename - end - - token - end - - def parse_DEPENDENCIES # :nodoc: - while !@tokens.empty? && peek.type == :text do - token = get :text - - requirements = [] - - case peek[0] - when :bang then - get :bang - - requirements << pinned_requirement(token.value) - when :l_paren then - get :l_paren - - loop do - op = get(:requirement).value - version = get(:text).value - - requirements << "#{op} #{version}" - - break unless peek.type == :comma - - get :comma - end - - get :r_paren - - if peek[0] == :bang - requirements.clear - requirements << pinned_requirement(token.value) - - get :bang - end - end - - @set.gem token.value, *requirements - - skip :newline - end - end - - def parse_GEM # :nodoc: - sources = [] - - while peek.first(2) == [:entry, "remote"] do - get :entry, "remote" - data = get(:text).value - skip :newline - - sources << Gem::Source.new(data) - end - - sources << Gem::Source.new(Gem::DEFAULT_HOST) if sources.empty? - - get :entry, "specs" - - skip :newline - - set = Gem::Resolver::LockSet.new sources - last_specs = nil - - while !@tokens.empty? && peek.type == :text do - token = get :text - name = token.value - column = token.column - - case peek[0] - when :newline then - last_specs.each do |spec| - spec.add_dependency Gem::Dependency.new name if column == 6 - end - when :l_paren then - get :l_paren - - token = get [:text, :requirement] - type = token.type - data = token.value - - if type == :text && column == 4 - version, platform = data.split "-", 2 - - platform = - platform ? Gem::Platform.new(platform) : Gem::Platform::RUBY - - last_specs = set.add name, version, platform - else - dependency = parse_dependency name, data - - last_specs.each do |spec| - spec.add_dependency dependency - end - end - - get :r_paren - else - raise "BUG: unknown token #{peek}" - end - - skip :newline - end - - @set.sets << set - end - - def parse_GIT # :nodoc: - get :entry, "remote" - repository = get(:text).value - - skip :newline - - get :entry, "revision" - revision = get(:text).value - - skip :newline - - type = peek.type - value = peek.value - if type == :entry && %w[branch ref tag].include?(value) - get - get :text - - skip :newline - end - - get :entry, "specs" - - skip :newline - - set = Gem::Resolver::GitSet.new - set.root_dir = @set.install_dir - - last_spec = nil - - while !@tokens.empty? && peek.type == :text do - token = get :text - name = token.value - column = token.column - - case peek[0] - when :newline then - last_spec.add_dependency Gem::Dependency.new name if column == 6 - when :l_paren then - get :l_paren - - token = get [:text, :requirement] - type = token.type - data = token.value - - if type == :text && column == 4 - last_spec = set.add_git_spec name, data, repository, revision, true - else - dependency = parse_dependency name, data - - last_spec.add_dependency dependency - end - - get :r_paren - else - raise "BUG: unknown token #{peek}" - end - - skip :newline - end - - @set.sets << set - end - - def parse_PATH # :nodoc: - get :entry, "remote" - directory = get(:text).value - - skip :newline - - get :entry, "specs" - - skip :newline - - set = Gem::Resolver::VendorSet.new - last_spec = nil - - while !@tokens.empty? && peek.first == :text do - token = get :text - name = token.value - column = token.column - - case peek[0] - when :newline then - last_spec.add_dependency Gem::Dependency.new name if column == 6 - when :l_paren then - get :l_paren - - token = get [:text, :requirement] - type = token.type - data = token.value - - if type == :text && column == 4 - last_spec = set.add_vendor_gem name, directory - else - dependency = parse_dependency name, data - - last_spec.dependencies << dependency - end - - get :r_paren - else - raise "BUG: unknown token #{peek}" - end - - skip :newline - end - - @set.sets << set - end - - def parse_PLATFORMS # :nodoc: - while !@tokens.empty? && peek.first == :text do - name = get(:text).value - - @platforms << name - - skip :newline - end - end - - ## - # Parses the requirements following the dependency +name+ and the +op+ for - # the first token of the requirements and returns a Gem::Dependency object. - - def parse_dependency(name, op) # :nodoc: - return Gem::Dependency.new name, op unless peek[0] == :text - - version = get(:text).value - - requirements = ["#{op} #{version}"] - - while peek.type == :comma do - get :comma - op = get(:requirement).value - version = get(:text).value - - requirements << "#{op} #{version}" - end - - Gem::Dependency.new name, requirements - end - - private - - def skip(type) # :nodoc: - @tokens.skip type - end - - ## - # Peeks at the next token for Lockfile - - def peek # :nodoc: - @tokens.peek - end - - def pinned_requirement(name) # :nodoc: - requirement = Gem::Dependency.new name - specification = @set.sets.flat_map do |set| - set.find_all(requirement) - end.compact.first - - specification&.version - end - - ## - # Ungets the last token retrieved by #get - - def unget(token) # :nodoc: - @tokens.unshift token - end -end diff --git a/lib/rubygems/request_set/lockfile/tokenizer.rb b/lib/rubygems/request_set/lockfile/tokenizer.rb deleted file mode 100644 index 65cef3baa08c..000000000000 --- a/lib/rubygems/request_set/lockfile/tokenizer.rb +++ /dev/null @@ -1,122 +0,0 @@ -# frozen_string_literal: true - -# ) frozen_string_literal: true -require_relative "parser" - -class Gem::RequestSet::Lockfile::Tokenizer - Token = Struct.new :type, :value, :column, :line - EOF = Token.new :EOF - - def self.from_file(file) - new File.read(file), file - end - - def initialize(input, filename = nil, line = 0, pos = 0) - @line = line - @line_pos = pos - @tokens = [] - @filename = filename - tokenize input - end - - def make_parser(set, platforms) - Gem::RequestSet::Lockfile::Parser.new self, set, platforms, @filename - end - - def to_a - @tokens.map {|token| [token.type, token.value, token.column, token.line] } - end - - def skip(type) - @tokens.shift while !@tokens.empty? && peek.type == type - end - - ## - # Calculates the column (by byte) and the line of the current token based on - # +byte_offset+. - - def token_pos(byte_offset) # :nodoc: - [byte_offset - @line_pos, @line] - end - - def empty? - @tokens.empty? - end - - def unshift(token) - @tokens.unshift token - end - - def next_token - @tokens.shift - end - alias_method :shift, :next_token - - def peek - @tokens.first || EOF - end - - private - - def tokenize(input) - require "strscan" - s = StringScanner.new input - - until s.eos? do - pos = s.pos - - pos = s.pos if leading_whitespace = s.scan(/ +/) - - if s.scan(/[<|=>]{7}/) - message = "your #{@filename} contains merge conflict markers" - column, line = token_pos pos - - raise Gem::RequestSet::Lockfile::ParseError.new message, column, line, @filename - end - - @tokens << - if s.scan(/\r?\n/) - - token = Token.new(:newline, nil, *token_pos(pos)) - @line_pos = s.pos - @line += 1 - token - elsif s.scan(/[A-Z]+/) - - if leading_whitespace - text = s.matched - text += s.scan(/[^\s)]*/).to_s # in case of no match - Token.new(:text, text, *token_pos(pos)) - else - Token.new(:section, s.matched, *token_pos(pos)) - end - elsif s.scan(/([a-z]+):\s/) - - s.pos -= 1 # rewind for possible newline - Token.new(:entry, s[1], *token_pos(pos)) - elsif s.scan(/\(/) - - Token.new(:l_paren, nil, *token_pos(pos)) - elsif s.scan(/\)/) - - Token.new(:r_paren, nil, *token_pos(pos)) - elsif s.scan(/<=|>=|=|~>|<|>|!=/) - - Token.new(:requirement, s.matched, *token_pos(pos)) - elsif s.scan(/,/) - - Token.new(:comma, nil, *token_pos(pos)) - elsif s.scan(/!/) - - Token.new(:bang, nil, *token_pos(pos)) - elsif s.scan(/[^\s),!]*/) - - Token.new(:text, s.matched, *token_pos(pos)) - else - raise "BUG: can't create token for: #{s.string[s.pos..-1].inspect}" - end - end - - @tokens - end -end From a8627d7634ed848f51670ef6729cef03b04b1281 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 26 May 2026 17:34:06 +0900 Subject: [PATCH 6/6] bin/rake update_manifest --- Manifest.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/Manifest.txt b/Manifest.txt index 72b806e54974..d29c736903cd 100644 --- a/Manifest.txt +++ b/Manifest.txt @@ -462,8 +462,6 @@ lib/rubygems/request/https_pool.rb lib/rubygems/request_set.rb lib/rubygems/request_set/gem_dependency_api.rb lib/rubygems/request_set/lockfile.rb -lib/rubygems/request_set/lockfile/parser.rb -lib/rubygems/request_set/lockfile/tokenizer.rb lib/rubygems/requirement.rb lib/rubygems/resolver.rb lib/rubygems/resolver/activation_request.rb