From 4817166e54ad98f9b3e9d06e9e8c7ccff992a957 Mon Sep 17 00:00:00 2001 From: Samuel Giddins Date: Fri, 20 Oct 2023 17:56:22 -0700 Subject: [PATCH] [rubygems/rubygems] Extract generate_index command to rubygems-generate_index gem So generate_index can be implemented with dependencies, such as the compact index Took this approach from feedback in https://github.com/rubygems/rubygems/pull/6853 Running `gem generate_index` by default will use an installed rubygems-generate_index, or install and then use the command from the gem Apply suggestions from code review https://github.com/rubygems/rubygems/commit/fc1cb9bc9e Co-authored-by: Hiroshi SHIBATA --- lib/rubygems/command_manager.rb | 1 + .../commands/generate_index_command.rb | 113 ++--- lib/rubygems/commands/help_command.rb | 2 +- lib/rubygems/indexer.rb | 429 ------------------ spec/bundler/commands/install_spec.rb | 2 +- .../install/gems/compact_index_spec.rb | 2 +- spec/bundler/support/builders.rb | 25 +- ...est_gem_commands_generate_index_command.rb | 81 ---- test/rubygems/test_gem_indexer.rb | 380 ---------------- test/rubygems/test_gem_source.rb | 4 +- 10 files changed, 60 insertions(+), 979 deletions(-) delete mode 100644 lib/rubygems/indexer.rb delete mode 100644 test/rubygems/test_gem_commands_generate_index_command.rb delete mode 100644 test/rubygems/test_gem_indexer.rb diff --git a/lib/rubygems/command_manager.rb b/lib/rubygems/command_manager.rb index c53f0231afc9f8..1028283ffa98fc 100644 --- a/lib/rubygems/command_manager.rb +++ b/lib/rubygems/command_manager.rb @@ -249,6 +249,7 @@ def load_and_instantiate(command_name) def invoke_command(args, build_args) cmd_name = args.shift.downcase cmd = find_command cmd_name + terminate_interaction 1 unless cmd cmd.deprecation_warning if cmd.deprecated? cmd.invoke_with_build_args args, build_args end diff --git a/lib/rubygems/commands/generate_index_command.rb b/lib/rubygems/commands/generate_index_command.rb index 0bf74355ae30c3..13be92593b1c47 100644 --- a/lib/rubygems/commands/generate_index_command.rb +++ b/lib/rubygems/commands/generate_index_command.rb @@ -1,86 +1,51 @@ # frozen_string_literal: true require_relative "../command" -require_relative "../indexer" -## -# Generates a index files for use as a gem server. -# -# See `gem help generate_index` - -class Gem::Commands::GenerateIndexCommand < Gem::Command - def initialize - super "generate_index", - "Generates the index files for a gem server directory", - directory: ".", build_modern: true +unless defined? Gem::Commands::GenerateIndexCommand + class Gem::Commands::GenerateIndexCommand < Gem::Command + module RubygemsTrampoline + def description # :nodoc: + <<~EOF + The generate_index command has been moved to the rubygems-generate_index gem. + EOF + end - add_option "-d", "--directory=DIRNAME", - "repository base dir containing gems subdir" do |dir, options| - options[:directory] = File.expand_path dir - end + def execute + alert_error "Install the rubygems-generate_index gem for the generate_index command" + end - add_option "--[no-]modern", - "Generate indexes for RubyGems", - "(always true)" do |value, options| - options[:build_modern] = value + def invoke_with_build_args(args, build_args) + name = "rubygems-generate_index" + spec = begin + Gem::Specification.find_by_name(name) + rescue Gem::LoadError + require "rubygems/dependency_installer" + Gem.install(name, Gem::Requirement.default, Gem::DependencyInstaller::DEFAULT_OPTIONS).find {|s| s.name == name } + end + + # remove the methods defined in this file so that the methods defined in the gem are used instead, + # and without a method redefinition warning + %w[description execute invoke_with_build_args].each do |method| + RubygemsTrampoline.remove_method(method) + end + self.class.singleton_class.remove_method(:new) + + spec.activate + Gem.load_plugin_files spec.matches_for_glob("rubygems_plugin#{Gem.suffix_pattern}") + + self.class.new.invoke_with_build_args(args, build_args) + end end + private_constant :RubygemsTrampoline - deprecate_option("--modern", version: "4.0", extra_msg: "Modern indexes (specs, latest_specs, and prerelease_specs) are always generated, so this option is not needed.") - deprecate_option("--no-modern", version: "4.0", extra_msg: "The `--no-modern` option is currently ignored. Modern indexes (specs, latest_specs, and prerelease_specs) are always generated.") - - add_option "--update", - "Update modern indexes with gems added", - "since the last update" do |value, options| - options[:update] = value + # remove_method(:initialize) warns, but removing new does not warn + def self.new + command = allocate + command.send(:initialize, "generate_index", "Generates the index files for a gem server directory (requires rubygems-generate_index)") + command end - end - - def defaults_str # :nodoc: - "--directory . --modern" - end - - def description # :nodoc: - <<-EOF -The generate_index command creates a set of indexes for serving gems -statically. The command expects a 'gems' directory under the path given to -the --directory option. The given directory will be the directory you serve -as the gem repository. -For `gem generate_index --directory /path/to/repo`, expose /path/to/repo via -your HTTP server configuration (not /path/to/repo/gems). - -When done, it will generate a set of files like this: - - gems/*.gem # .gem files you want to - # index - - specs..gz # specs index - latest_specs..gz # latest specs index - prerelease_specs..gz # prerelease specs index - quick/Marshal./.gemspec.rz # Marshal quick index file - -The .rz extension files are compressed with the inflate algorithm. -The Marshal version number comes from ruby's Marshal::MAJOR_VERSION and -Marshal::MINOR_VERSION constants. It is used to ensure compatibility. - EOF - end - - def execute - # This is always true because it's the only way now. - options[:build_modern] = true - - if !File.exist?(options[:directory]) || - !File.directory?(options[:directory]) - alert_error "unknown directory name #{options[:directory]}." - terminate_interaction 1 - else - indexer = Gem::Indexer.new options.delete(:directory), options - - if options[:update] - indexer.update_index - else - indexer.generate_index - end - end + prepend(RubygemsTrampoline) end end diff --git a/lib/rubygems/commands/help_command.rb b/lib/rubygems/commands/help_command.rb index 043e7d36911342..8994f1aa098da5 100644 --- a/lib/rubygems/commands/help_command.rb +++ b/lib/rubygems/commands/help_command.rb @@ -333,7 +333,7 @@ def show_commands # :nodoc: @command_manager.command_names.each do |cmd_name| command = @command_manager[cmd_name] - next if command.deprecated? + next if command&.deprecated? summary = if command diff --git a/lib/rubygems/indexer.rb b/lib/rubygems/indexer.rb deleted file mode 100644 index 82c334672f60e2..00000000000000 --- a/lib/rubygems/indexer.rb +++ /dev/null @@ -1,429 +0,0 @@ -# frozen_string_literal: true - -require_relative "../rubygems" -require_relative "package" -require "tmpdir" - -## -# Top level class for building the gem repository index. - -class Gem::Indexer - include Gem::UserInteraction - - ## - # Build indexes for RubyGems 1.2.0 and newer when true - - attr_accessor :build_modern - - ## - # Index install location - - attr_reader :dest_directory - - ## - # Specs index install location - - attr_reader :dest_specs_index - - ## - # Latest specs index install location - - attr_reader :dest_latest_specs_index - - ## - # Prerelease specs index install location - - attr_reader :dest_prerelease_specs_index - - ## - # Index build directory - - attr_reader :directory - - ## - # Create an indexer that will index the gems in +directory+. - - def initialize(directory, options = {}) - require "fileutils" - require "tmpdir" - require "zlib" - - options = { build_modern: true }.merge options - - @build_modern = options[:build_modern] - - @dest_directory = directory - @directory = Dir.mktmpdir "gem_generate_index" - - marshal_name = "Marshal.#{Gem.marshal_version}" - - @master_index = File.join @directory, "yaml" - @marshal_index = File.join @directory, marshal_name - - @quick_dir = File.join @directory, "quick" - @quick_marshal_dir = File.join @quick_dir, marshal_name - @quick_marshal_dir_base = File.join "quick", marshal_name # FIX: UGH - - @quick_index = File.join @quick_dir, "index" - @latest_index = File.join @quick_dir, "latest_index" - - @specs_index = File.join @directory, "specs.#{Gem.marshal_version}" - @latest_specs_index = - File.join(@directory, "latest_specs.#{Gem.marshal_version}") - @prerelease_specs_index = - File.join(@directory, "prerelease_specs.#{Gem.marshal_version}") - @dest_specs_index = - File.join(@dest_directory, "specs.#{Gem.marshal_version}") - @dest_latest_specs_index = - File.join(@dest_directory, "latest_specs.#{Gem.marshal_version}") - @dest_prerelease_specs_index = - File.join(@dest_directory, "prerelease_specs.#{Gem.marshal_version}") - - @files = [] - end - - ## - # Build various indices - - def build_indices - specs = map_gems_to_specs gem_file_list - Gem::Specification._resort! specs - build_marshal_gemspecs specs - build_modern_indices specs if @build_modern - - compress_indices - end - - ## - # Builds Marshal quick index gemspecs. - - def build_marshal_gemspecs(specs) - count = specs.count - progress = ui.progress_reporter count, - "Generating Marshal quick index gemspecs for #{count} gems", - "Complete" - - files = [] - - Gem.time "Generated Marshal quick index gemspecs" do - specs.each do |spec| - next if spec.default_gem? - spec_file_name = "#{spec.original_name}.gemspec.rz" - marshal_name = File.join @quick_marshal_dir, spec_file_name - - marshal_zipped = Gem.deflate Marshal.dump(spec) - - File.open marshal_name, "wb" do |io| - io.write marshal_zipped - end - - files << marshal_name - - progress.updated spec.original_name - end - - progress.done - end - - @files << @quick_marshal_dir - - files - end - - ## - # Build a single index for RubyGems 1.2 and newer - - def build_modern_index(index, file, name) - say "Generating #{name} index" - - Gem.time "Generated #{name} index" do - File.open(file, "wb") do |io| - specs = index.map do |*spec| - # We have to splat here because latest_specs is an array, while the - # others are hashes. - spec = spec.flatten.last - platform = spec.original_platform - - # win32-api-1.0.4-x86-mswin32-60 - unless String === platform - alert_warning "Skipping invalid platform in gem: #{spec.full_name}" - next - end - - platform = Gem::Platform::RUBY if platform.nil? || platform.empty? - [spec.name, spec.version, platform] - end - - specs = compact_specs(specs) - Marshal.dump(specs, io) - end - end - end - - ## - # Builds indices for RubyGems 1.2 and newer. Handles full, latest, prerelease - - def build_modern_indices(specs) - prerelease, released = specs.partition do |s| - s.version.prerelease? - end - latest_specs = - Gem::Specification._latest_specs specs - - build_modern_index(released.sort, @specs_index, "specs") - build_modern_index(latest_specs.sort, @latest_specs_index, "latest specs") - build_modern_index(prerelease.sort, @prerelease_specs_index, - "prerelease specs") - - @files += [@specs_index, - "#{@specs_index}.gz", - @latest_specs_index, - "#{@latest_specs_index}.gz", - @prerelease_specs_index, - "#{@prerelease_specs_index}.gz"] - end - - def map_gems_to_specs(gems) - gems.map do |gemfile| - if File.size(gemfile) == 0 - alert_warning "Skipping zero-length gem: #{gemfile}" - next - end - - begin - spec = Gem::Package.new(gemfile).spec - spec.loaded_from = gemfile - - spec.abbreviate - spec.sanitize - - spec - rescue SignalException - alert_error "Received signal, exiting" - raise - rescue StandardError => e - msg = ["Unable to process #{gemfile}", - "#{e.message} (#{e.class})", - "\t#{e.backtrace.join "\n\t"}"].join("\n") - alert_error msg - end - end.compact - end - - ## - # Compresses indices on disk - #-- - # All future files should be compressed using gzip, not deflate - - def compress_indices - say "Compressing indices" - - Gem.time "Compressed indices" do - if @build_modern - gzip @specs_index - gzip @latest_specs_index - gzip @prerelease_specs_index - end - end - end - - ## - # Compacts Marshal output for the specs index data source by using identical - # objects as much as possible. - - def compact_specs(specs) - names = {} - versions = {} - platforms = {} - - specs.map do |(name, version, platform)| - names[name] = name unless names.include? name - versions[version] = version unless versions.include? version - platforms[platform] = platform unless platforms.include? platform - - [names[name], versions[version], platforms[platform]] - end - end - - ## - # Compress +filename+ with +extension+. - - def compress(filename, extension) - data = Gem.read_binary filename - - zipped = Gem.deflate data - - File.open "#{filename}.#{extension}", "wb" do |io| - io.write zipped - end - end - - ## - # List of gem file names to index. - - def gem_file_list - Gem::Util.glob_files_in_dir("*.gem", File.join(@dest_directory, "gems")) - end - - ## - # Builds and installs indices. - - def generate_index - make_temp_directories - build_indices - install_indices - rescue SignalException - ensure - FileUtils.rm_rf @directory - end - - ## - # Zlib::GzipWriter wrapper that gzips +filename+ on disk. - - def gzip(filename) - Zlib::GzipWriter.open "#{filename}.gz" do |io| - io.write Gem.read_binary(filename) - end - end - - ## - # Install generated indices into the destination directory. - - def install_indices - verbose = Gem.configuration.really_verbose - - say "Moving index into production dir #{@dest_directory}" if verbose - - files = @files - files.delete @quick_marshal_dir if files.include? @quick_dir - - if files.include?(@quick_marshal_dir) && !files.include?(@quick_dir) - files.delete @quick_marshal_dir - - dst_name = File.join(@dest_directory, @quick_marshal_dir_base) - - FileUtils.mkdir_p File.dirname(dst_name), verbose: verbose - FileUtils.rm_rf dst_name, verbose: verbose - FileUtils.mv(@quick_marshal_dir, dst_name, - verbose: verbose, force: true) - end - - files = files.map do |path| - path.sub(%r{^#{Regexp.escape @directory}/?}, "") # HACK? - end - - files.each do |file| - src_name = File.join @directory, file - dst_name = File.join @dest_directory, file - - FileUtils.rm_rf dst_name, verbose: verbose - FileUtils.mv(src_name, @dest_directory, - verbose: verbose, force: true) - end - end - - ## - # Make directories for index generation - - def make_temp_directories - FileUtils.rm_rf @directory - FileUtils.mkdir_p @directory, mode: 0o700 - FileUtils.mkdir_p @quick_marshal_dir - end - - ## - # Ensure +path+ and path with +extension+ are identical. - - def paranoid(path, extension) - data = Gem.read_binary path - compressed_data = Gem.read_binary "#{path}.#{extension}" - - unless data == Gem::Util.inflate(compressed_data) - raise "Compressed file #{compressed_path} does not match uncompressed file #{path}" - end - end - - ## - # Perform an in-place update of the repository from newly added gems. - - def update_index - make_temp_directories - - specs_mtime = File.stat(@dest_specs_index).mtime - newest_mtime = Time.at 0 - - updated_gems = gem_file_list.select do |gem| - gem_mtime = File.stat(gem).mtime - newest_mtime = gem_mtime if gem_mtime > newest_mtime - gem_mtime >= specs_mtime - end - - if updated_gems.empty? - say "No new gems" - terminate_interaction 0 - end - - specs = map_gems_to_specs updated_gems - prerelease, released = specs.partition {|s| s.version.prerelease? } - - files = build_marshal_gemspecs specs - - Gem.time "Updated indexes" do - update_specs_index released, @dest_specs_index, @specs_index - update_specs_index released, @dest_latest_specs_index, @latest_specs_index - update_specs_index(prerelease, - @dest_prerelease_specs_index, - @prerelease_specs_index) - end - - compress_indices - - verbose = Gem.configuration.really_verbose - - say "Updating production dir #{@dest_directory}" if verbose - - files << @specs_index - files << "#{@specs_index}.gz" - files << @latest_specs_index - files << "#{@latest_specs_index}.gz" - files << @prerelease_specs_index - files << "#{@prerelease_specs_index}.gz" - - files = files.map do |path| - path.sub(%r{^#{Regexp.escape @directory}/?}, "") # HACK? - end - - files.each do |file| - src_name = File.join @directory, file - dst_name = File.join @dest_directory, file # REFACTOR: duped above - - FileUtils.mv src_name, dst_name, verbose: verbose, - force: true - - File.utime newest_mtime, newest_mtime, dst_name - end - ensure - FileUtils.rm_rf @directory - end - - ## - # Combines specs in +index+ and +source+ then writes out a new copy to - # +dest+. For a latest index, does not ensure the new file is minimal. - - def update_specs_index(index, source, dest) - Gem.load_safe_marshal - specs_index = Gem::SafeMarshal.safe_load Gem.read_binary(source) - - index.each do |spec| - platform = spec.original_platform - platform = Gem::Platform::RUBY if platform.nil? || platform.empty? - specs_index << [spec.name, spec.version, platform] - end - - specs_index = compact_specs specs_index.uniq.sort - - File.open dest, "wb" do |io| - Marshal.dump specs_index, io - end - end -end diff --git a/spec/bundler/commands/install_spec.rb b/spec/bundler/commands/install_spec.rb index 5f5c9bb28642c3..aa7c54ce4b43fd 100644 --- a/spec/bundler/commands/install_spec.rb +++ b/spec/bundler/commands/install_spec.rb @@ -556,7 +556,7 @@ end it "fails gracefully when downloading an invalid specification from the full index" do - build_repo2 do + build_repo2(build_compact_index: false) do build_gem "ajp-rails", "0.0.0", gemspec: false, skip_validation: true do |s| bad_deps = [["ruby-ajp", ">= 0.2.0"], ["rails", ">= 0.14"]] s. diff --git a/spec/bundler/install/gems/compact_index_spec.rb b/spec/bundler/install/gems/compact_index_spec.rb index 51d7af9ee24ad2..f3b8bb708dc902 100644 --- a/spec/bundler/install/gems/compact_index_spec.rb +++ b/spec/bundler/install/gems/compact_index_spec.rb @@ -44,7 +44,7 @@ end it "should handle case sensitivity conflicts" do - build_repo4 do + build_repo4(build_compact_index: false) do build_gem "rack", "1.0" do |s| s.add_runtime_dependency("Rack", "0.1") end diff --git a/spec/bundler/support/builders.rb b/spec/bundler/support/builders.rb index 76e05b520327e2..d36c5d47173a71 100644 --- a/spec/bundler/support/builders.rb +++ b/spec/bundler/support/builders.rb @@ -191,25 +191,25 @@ def build_repo1 end end - def build_repo2(&blk) + def build_repo2(**kwargs, &blk) FileUtils.rm_rf gem_repo2 FileUtils.cp_r gem_repo1, gem_repo2 - update_repo2(&blk) if block_given? + update_repo2(**kwargs, &blk) if block_given? end # A repo that has no pre-installed gems included. (The caller completely # determines the contents with the block.) - def build_repo4(&blk) + def build_repo4(**kwargs, &blk) FileUtils.rm_rf gem_repo4 - build_repo(gem_repo4, &blk) + build_repo(gem_repo4, **kwargs, &blk) end def update_repo4(&blk) update_repo(gem_repo4, &blk) end - def update_repo2(&blk) - update_repo(gem_repo2, &blk) + def update_repo2(**kwargs, &blk) + update_repo(gem_repo2, **kwargs, &blk) end def build_security_repo @@ -227,12 +227,12 @@ def build_security_repo end end - def build_repo(path, &blk) + def build_repo(path, **kwargs, &blk) return if File.directory?(path) FileUtils.mkdir_p("#{path}/gems") - update_repo(path, &blk) + update_repo(path,**kwargs, &blk) end def check_test_gems! @@ -249,7 +249,7 @@ def check_test_gems! end end - def update_repo(path) + def update_repo(path, build_compact_index: true) if path == gem_repo1 && caller.first.split(" ").last == "`build_repo`" raise "Updating gem_repo1 is unsupported -- use gem_repo2 instead" end @@ -258,7 +258,12 @@ def update_repo(path) @_build_repo = File.basename(path) yield with_gem_path_as Path.base_system_gem_path do - gem_command :generate_index, dir: path + Dir[Spec::Path.base_system_gem_path.join("gems/rubygems-generate_index*/lib")].first || + raise("Could not find rubygems-generate_index lib directory in #{Spec::Path.base_system_gem_path}") + + command = "generate_index" + command += " --no-compact" if !build_compact_index && gem_command(command + " --help").include?("--[no-]compact") + gem_command command, dir: path end ensure @_build_path = nil diff --git a/test/rubygems/test_gem_commands_generate_index_command.rb b/test/rubygems/test_gem_commands_generate_index_command.rb deleted file mode 100644 index 7e719a5f60c214..00000000000000 --- a/test/rubygems/test_gem_commands_generate_index_command.rb +++ /dev/null @@ -1,81 +0,0 @@ -# frozen_string_literal: true - -require_relative "helper" -require "rubygems/indexer" -require "rubygems/commands/generate_index_command" - -class TestGemCommandsGenerateIndexCommand < Gem::TestCase - def setup - super - - @cmd = Gem::Commands::GenerateIndexCommand.new - @cmd.options[:directory] = @gemhome - end - - def test_execute - use_ui @ui do - @cmd.execute - end - - specs = File.join @gemhome, "specs.4.8.gz" - - assert File.exist?(specs), specs - end - - def test_execute_no_modern - @cmd.options[:modern] = false - - use_ui @ui do - @cmd.execute - end - - specs = File.join @gemhome, "specs.4.8.gz" - - assert File.exist?(specs), specs - end - - def test_handle_options_directory - return if Gem.win_platform? - refute_equal "/nonexistent", @cmd.options[:directory] - - @cmd.handle_options %w[--directory /nonexistent] - - assert_equal "/nonexistent", @cmd.options[:directory] - end - - def test_handle_options_directory_windows - return unless Gem.win_platform? - - refute_equal "/nonexistent", @cmd.options[:directory] - - @cmd.handle_options %w[--directory C:/nonexistent] - - assert_equal "C:/nonexistent", @cmd.options[:directory] - end - - def test_handle_options_update - @cmd.handle_options %w[--update] - - assert @cmd.options[:update] - end - - def test_handle_options_modern - use_ui @ui do - @cmd.handle_options %w[--modern] - end - - assert_equal \ - "WARNING: The \"--modern\" option has been deprecated and will be removed in Rubygems 4.0. Modern indexes (specs, latest_specs, and prerelease_specs) are always generated, so this option is not needed.\n", - @ui.error - end - - def test_handle_options_no_modern - use_ui @ui do - @cmd.handle_options %w[--no-modern] - end - - assert_equal \ - "WARNING: The \"--no-modern\" option has been deprecated and will be removed in Rubygems 4.0. The `--no-modern` option is currently ignored. Modern indexes (specs, latest_specs, and prerelease_specs) are always generated.\n", - @ui.error - end -end diff --git a/test/rubygems/test_gem_indexer.rb b/test/rubygems/test_gem_indexer.rb deleted file mode 100644 index 111c9154fe3e92..00000000000000 --- a/test/rubygems/test_gem_indexer.rb +++ /dev/null @@ -1,380 +0,0 @@ -# frozen_string_literal: true - -require_relative "helper" -require "rubygems/indexer" - -class TestGemIndexer < Gem::TestCase - def setup - super - - util_make_gems - - @d2_0 = util_spec "d", "2.0" do |s| - s.date = Gem::Specification::TODAY - 86_400 * 3 - end - util_build_gem @d2_0 - - @d2_0_a = util_spec "d", "2.0.a" - util_build_gem @d2_0_a - - @d2_0_b = util_spec "d", "2.0.b" - util_build_gem @d2_0_b - - @default = new_default_spec "default", 2 - install_default_gems @default - - @indexerdir = File.join(@tempdir, "indexer") - - gems = File.join(@indexerdir, "gems") - FileUtils.mkdir_p gems - FileUtils.mv Dir[File.join(@gemhome, "cache", "*.gem")], gems - - @indexer = Gem::Indexer.new(@indexerdir) - end - - def teardown - FileUtils.rm_rf(@indexer.directory) - ensure - super - end - - def with_indexer(dir, **opts) - indexer = Gem::Indexer.new(dir, **opts) - build_directory = indexer.directory - yield indexer - ensure - FileUtils.rm_rf(build_directory) if build_directory - end - - def test_initialize - assert_equal @indexerdir, @indexer.dest_directory - Dir.mktmpdir("gem_generate_index") do |tmpdir| - assert_match(%r{#{tmpdir.match(/.*-/)}}, @indexer.directory) # rubocop:disable Style/RegexpLiteral - end - - with_indexer(@indexerdir) do |indexer| - assert_predicate indexer, :build_modern - end - - with_indexer(@indexerdir, build_modern: true) do |indexer| - assert_predicate indexer, :build_modern - end - end - - def test_build_indices - @indexer.make_temp_directories - - use_ui @ui do - @indexer.build_indices - end - - specs_path = File.join @indexer.directory, "specs.#{@marshal_version}" - specs_dump = Gem.read_binary specs_path - specs = Marshal.load specs_dump - - expected = [["a", Gem::Version.new("1"), "ruby"], - ["a", Gem::Version.new("2"), "ruby"], - ["a_evil", Gem::Version.new("9"), "ruby"], - ["b", Gem::Version.new("2"), "ruby"], - ["c", Gem::Version.new("1.2"), "ruby"], - ["d", Gem::Version.new("2.0"), "ruby"], - ["dep_x", Gem::Version.new("1"), "ruby"], - ["pl", Gem::Version.new("1"), "i386-linux"], - ["x", Gem::Version.new("1"), "ruby"]] - - assert_equal expected, specs - - latest_specs_path = File.join(@indexer.directory, - "latest_specs.#{@marshal_version}") - latest_specs_dump = Gem.read_binary latest_specs_path - latest_specs = Marshal.load latest_specs_dump - - expected = [["a", Gem::Version.new("2"), "ruby"], - ["a_evil", Gem::Version.new("9"), "ruby"], - ["b", Gem::Version.new("2"), "ruby"], - ["c", Gem::Version.new("1.2"), "ruby"], - ["d", Gem::Version.new("2.0"), "ruby"], - ["dep_x", Gem::Version.new("1"), "ruby"], - ["pl", Gem::Version.new("1"), "i386-linux"], - ["x", Gem::Version.new("1"), "ruby"]] - - assert_equal expected, latest_specs, "latest_specs" - end - - def test_generate_index - use_ui @ui do - @indexer.generate_index - end - - quickdir = File.join @indexerdir, "quick" - marshal_quickdir = File.join quickdir, "Marshal.#{@marshal_version}" - - assert_directory_exists quickdir - assert_directory_exists marshal_quickdir - - assert_indexed marshal_quickdir, "#{File.basename(@a1.spec_file)}.rz" - assert_indexed marshal_quickdir, "#{File.basename(@a2.spec_file)}.rz" - - refute_indexed marshal_quickdir, File.basename(@c1_2.spec_file) - - assert_indexed @indexerdir, "specs.#{@marshal_version}" - assert_indexed @indexerdir, "specs.#{@marshal_version}.gz" - - assert_indexed @indexerdir, "latest_specs.#{@marshal_version}" - assert_indexed @indexerdir, "latest_specs.#{@marshal_version}.gz" - - refute_directory_exists @indexer.directory - end - - def test_generate_index_modern - @indexer.build_modern = true - - use_ui @ui do - @indexer.generate_index - end - - refute_indexed @indexerdir, "yaml" - refute_indexed @indexerdir, "yaml.Z" - refute_indexed @indexerdir, "Marshal.#{@marshal_version}" - refute_indexed @indexerdir, "Marshal.#{@marshal_version}.Z" - - quickdir = File.join @indexerdir, "quick" - marshal_quickdir = File.join quickdir, "Marshal.#{@marshal_version}" - - assert_directory_exists quickdir, "quickdir should be directory" - assert_directory_exists marshal_quickdir - - refute_indexed quickdir, "index" - refute_indexed quickdir, "index.rz" - - refute_indexed quickdir, "latest_index" - refute_indexed quickdir, "latest_index.rz" - - refute_indexed quickdir, "#{File.basename(@a1.spec_file)}.rz" - refute_indexed quickdir, "#{File.basename(@a2.spec_file)}.rz" - refute_indexed quickdir, "#{File.basename(@b2.spec_file)}.rz" - refute_indexed quickdir, "#{File.basename(@c1_2.spec_file)}.rz" - - refute_indexed quickdir, "#{@pl1.original_name}.gemspec.rz" - refute_indexed quickdir, "#{File.basename(@pl1.spec_file)}.rz" - - assert_indexed marshal_quickdir, "#{File.basename(@a1.spec_file)}.rz" - assert_indexed marshal_quickdir, "#{File.basename(@a2.spec_file)}.rz" - - refute_indexed quickdir, File.basename(@c1_2.spec_file).to_s - refute_indexed marshal_quickdir, File.basename(@c1_2.spec_file).to_s - - assert_indexed @indexerdir, "specs.#{@marshal_version}" - assert_indexed @indexerdir, "specs.#{@marshal_version}.gz" - - assert_indexed @indexerdir, "latest_specs.#{@marshal_version}" - assert_indexed @indexerdir, "latest_specs.#{@marshal_version}.gz" - end - - def test_generate_index_modern_back_to_back - @indexer.build_modern = true - - use_ui @ui do - @indexer.generate_index - end - - with_indexer @indexerdir do |indexer| - indexer.build_modern = true - - use_ui @ui do - indexer.generate_index - end - quickdir = File.join @indexerdir, "quick" - marshal_quickdir = File.join quickdir, "Marshal.#{@marshal_version}" - - assert_directory_exists quickdir - assert_directory_exists marshal_quickdir - - assert_indexed marshal_quickdir, "#{File.basename(@a1.spec_file)}.rz" - assert_indexed marshal_quickdir, "#{File.basename(@a2.spec_file)}.rz" - - assert_indexed @indexerdir, "specs.#{@marshal_version}" - assert_indexed @indexerdir, "specs.#{@marshal_version}.gz" - - assert_indexed @indexerdir, "latest_specs.#{@marshal_version}" - assert_indexed @indexerdir, "latest_specs.#{@marshal_version}.gz" - end - end - - def test_generate_index_ui - use_ui @ui do - @indexer.generate_index - end - - assert_match(/^\.\.\.\.\.\.\.\.\.\.\.\.$/, @ui.output) - assert_match(/^Generating Marshal quick index gemspecs for 12 gems$/, @ui.output) - assert_match(/^Complete$/, @ui.output) - assert_match(/^Generating specs index$/, @ui.output) - assert_match(/^Generating latest specs index$/, @ui.output) - assert_match(/^Generating prerelease specs index$/, @ui.output) - assert_match(/^Complete$/, @ui.output) - assert_match(/^Compressing indices$/, @ui.output) - - assert_equal "", @ui.error - end - - def test_generate_index_specs - use_ui @ui do - @indexer.generate_index - end - - specs_path = File.join @indexerdir, "specs.#{@marshal_version}" - - specs_dump = Gem.read_binary specs_path - specs = Marshal.load specs_dump - - expected = [ - ["a", Gem::Version.new(1), "ruby"], - ["a", Gem::Version.new(2), "ruby"], - ["a_evil", Gem::Version.new(9), "ruby"], - ["b", Gem::Version.new(2), "ruby"], - ["c", Gem::Version.new("1.2"), "ruby"], - ["d", Gem::Version.new("2.0"), "ruby"], - ["dep_x", Gem::Version.new(1), "ruby"], - ["pl", Gem::Version.new(1), "i386-linux"], - ["x", Gem::Version.new(1), "ruby"], - ] - - assert_equal expected, specs - - assert_same specs[0].first, specs[1].first, - "identical names not identical" - - assert_same specs[0][1], specs[-1][1], - "identical versions not identical" - - assert_same specs[0].last, specs[1].last, - "identical platforms not identical" - - refute_same specs[1][1], specs[5][1], - "different versions not different" - end - - def test_generate_index_latest_specs - use_ui @ui do - @indexer.generate_index - end - - latest_specs_path = File.join @indexerdir, "latest_specs.#{@marshal_version}" - - latest_specs_dump = Gem.read_binary latest_specs_path - latest_specs = Marshal.load latest_specs_dump - - expected = [ - ["a", Gem::Version.new(2), "ruby"], - ["a_evil", Gem::Version.new(9), "ruby"], - ["b", Gem::Version.new(2), "ruby"], - ["c", Gem::Version.new("1.2"), "ruby"], - ["d", Gem::Version.new("2.0"), "ruby"], - ["dep_x", Gem::Version.new(1), "ruby"], - ["pl", Gem::Version.new(1), "i386-linux"], - ["x", Gem::Version.new(1), "ruby"], - ] - - assert_equal expected, latest_specs - - assert_same latest_specs[0][1], latest_specs[2][1], - "identical versions not identical" - - assert_same latest_specs[0].last, latest_specs[1].last, - "identical platforms not identical" - end - - def test_generate_index_prerelease_specs - use_ui @ui do - @indexer.generate_index - end - - prerelease_specs_path = File.join @indexerdir, "prerelease_specs.#{@marshal_version}" - - prerelease_specs_dump = Gem.read_binary prerelease_specs_path - prerelease_specs = Marshal.load prerelease_specs_dump - - assert_equal [["a", Gem::Version.new("3.a"), "ruby"], - ["d", Gem::Version.new("2.0.a"), "ruby"], - ["d", Gem::Version.new("2.0.b"), "ruby"]], - prerelease_specs - end - - ## - # Emulate the starting state of Gem::Specification in a live environment, - # where it will carry the list of system gems - def with_system_gems - Gem::Specification.reset - - sys_gem = util_spec "systemgem", "1.0" - util_build_gem sys_gem - install_default_gems sys_gem - yield - util_remove_gem sys_gem - end - - def test_update_index - use_ui @ui do - @indexer.generate_index - end - - quickdir = File.join @indexerdir, "quick" - marshal_quickdir = File.join quickdir, "Marshal.#{@marshal_version}" - - assert_directory_exists quickdir - assert_directory_exists marshal_quickdir - - @d2_1 = util_spec "d", "2.1" - util_build_gem @d2_1 - @d2_1_tuple = [@d2_1.name, @d2_1.version, @d2_1.original_platform] - - @d2_1_a = util_spec "d", "2.2.a" - util_build_gem @d2_1_a - @d2_1_a_tuple = [@d2_1_a.name, @d2_1_a.version, @d2_1_a.original_platform] - - gems = File.join @indexerdir, "gems" - - FileUtils.mv @d2_1.cache_file, gems - FileUtils.mv @d2_1_a.cache_file, gems - - with_system_gems do - use_ui @ui do - @indexer.update_index - end - - assert_indexed marshal_quickdir, "#{File.basename(@d2_1.spec_file)}.rz" - - specs_index = Marshal.load Gem.read_binary(@indexer.dest_specs_index) - - assert_includes specs_index, @d2_1_tuple - refute_includes specs_index, @d2_1_a_tuple - - latest_specs_index = Marshal.load \ - Gem.read_binary(@indexer.dest_latest_specs_index) - - assert_includes latest_specs_index, @d2_1_tuple - assert_includes latest_specs_index, - [@d2_0.name, @d2_0.version, @d2_0.original_platform] - refute_includes latest_specs_index, @d2_1_a_tuple - - pre_specs_index = Marshal.load \ - Gem.read_binary(@indexer.dest_prerelease_specs_index) - - assert_includes pre_specs_index, @d2_1_a_tuple - refute_includes pre_specs_index, @d2_1_tuple - - refute_directory_exists @indexer.directory - end - end - - def assert_indexed(dir, name) - file = File.join dir, name - assert File.exist?(file), "#{file} does not exist" - end - - def refute_indexed(dir, name) - file = File.join dir, name - refute File.exist?(file), "#{file} exists" - end -end diff --git a/test/rubygems/test_gem_source.rb b/test/rubygems/test_gem_source.rb index 0372c6253cc3c4..55cc50a5b2ecec 100644 --- a/test/rubygems/test_gem_source.rb +++ b/test/rubygems/test_gem_source.rb @@ -2,7 +2,6 @@ require_relative "helper" require "rubygems/source" -require "rubygems/indexer" class TestGemSource < Gem::TestCase def tuple(*args) @@ -55,7 +54,8 @@ def test_dependency_resolver_set_bundler_api end def test_dependency_resolver_set_file_uri - Gem::Indexer.new(@tempdir).generate_index + File.write(File.join(@tempdir, "prerelease_specs.4.8.gz"), Gem::Util.gzip("\x04\x08[\x05".b)) + File.write(File.join(@tempdir, "specs.4.8.gz"), Gem::Util.gzip("\x04\x08[\x05".b)) source = Gem::Source.new "file://#{@tempdir}/"