Refactor vendoring to allow validating vendoring is reproducible

Helps ensure that unsuspecting diffs to the vendored code arent accidentally introduced

.github/workflows/ubuntu-lint.yml

@@ -11,7 +11,7 @@ concurrency:
   group: ci-${{ github.ref }}-${{ github.workflow }}
   cancel-in-progress: true
 
-permissions:  # added using https://github.com/step-security/secure-workflows
+permissions: # added using https://github.com/step-security/secure-workflows
   contents: read
 
 jobs:
@@ -36,5 +36,5 @@ jobs:
       - name: Install & Check Dependencies
         run: bin/rake dev:frozen_deps
       - name: Misc checks
-        run: bin/rake check_rvm_integration man:check check_rubygems_integration
+        run: bin/rake check_rvm_integration man:check vendor:check check_rubygems_integration
     timeout-minutes: 15

Manifest.txt

@@ -248,18 +248,22 @@ bundler/lib/bundler/ui/silent.rb
 bundler/lib/bundler/uri_credentials_filter.rb
 bundler/lib/bundler/uri_normalizer.rb
 bundler/lib/bundler/vendor/.document
+bundler/lib/bundler/vendor/connection_pool/.document
 bundler/lib/bundler/vendor/connection_pool/LICENSE
 bundler/lib/bundler/vendor/connection_pool/lib/connection_pool.rb
 bundler/lib/bundler/vendor/connection_pool/lib/connection_pool/timed_stack.rb
 bundler/lib/bundler/vendor/connection_pool/lib/connection_pool/version.rb
 bundler/lib/bundler/vendor/connection_pool/lib/connection_pool/wrapper.rb
+bundler/lib/bundler/vendor/fileutils/.document
 bundler/lib/bundler/vendor/fileutils/LICENSE.txt
 bundler/lib/bundler/vendor/fileutils/lib/fileutils.rb
+bundler/lib/bundler/vendor/net-http-persistent/.document
 bundler/lib/bundler/vendor/net-http-persistent/README.rdoc
 bundler/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent.rb
 bundler/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/connection.rb
 bundler/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/pool.rb
 bundler/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/timed_stack_multi.rb
+bundler/lib/bundler/vendor/pub_grub/.document
 bundler/lib/bundler/vendor/pub_grub/LICENSE.txt
 bundler/lib/bundler/vendor/pub_grub/lib/pub_grub.rb
 bundler/lib/bundler/vendor/pub_grub/lib/pub_grub/assignment.rb
@@ -277,6 +281,7 @@ bundler/lib/bundler/vendor/pub_grub/lib/pub_grub/version_constraint.rb
 bundler/lib/bundler/vendor/pub_grub/lib/pub_grub/version_range.rb
 bundler/lib/bundler/vendor/pub_grub/lib/pub_grub/version_solver.rb
 bundler/lib/bundler/vendor/pub_grub/lib/pub_grub/version_union.rb
+bundler/lib/bundler/vendor/thor/.document
 bundler/lib/bundler/vendor/thor/LICENSE.md
 bundler/lib/bundler/vendor/thor/lib/thor.rb
 bundler/lib/bundler/vendor/thor/lib/thor/actions.rb
@@ -314,8 +319,10 @@ bundler/lib/bundler/vendor/thor/lib/thor/shell/terminal.rb
 bundler/lib/bundler/vendor/thor/lib/thor/shell/wrapped_printer.rb
 bundler/lib/bundler/vendor/thor/lib/thor/util.rb
 bundler/lib/bundler/vendor/thor/lib/thor/version.rb
+bundler/lib/bundler/vendor/tsort/.document
 bundler/lib/bundler/vendor/tsort/LICENSE.txt
 bundler/lib/bundler/vendor/tsort/lib/tsort.rb
+bundler/lib/bundler/vendor/uri/.document
 bundler/lib/bundler/vendor/uri/LICENSE.txt
 bundler/lib/bundler/vendor/uri/lib/uri.rb
 bundler/lib/bundler/vendor/uri/lib/uri/common.rb
@@ -425,6 +432,7 @@ lib/rubygems/installer.rb
 lib/rubygems/installer_uninstaller_utils.rb
 lib/rubygems/local_remote_options.rb
 lib/rubygems/name_tuple.rb
+lib/rubygems/net-http/.document
 lib/rubygems/net-http/LICENSE.txt
 lib/rubygems/net-http/lib/net/http.rb
 lib/rubygems/net-http/lib/net/http/backward.rb
@@ -438,6 +446,7 @@ lib/rubygems/net-http/lib/net/http/response.rb
 lib/rubygems/net-http/lib/net/http/responses.rb
 lib/rubygems/net-http/lib/net/http/status.rb
 lib/rubygems/net-http/lib/net/https.rb
+lib/rubygems/net-protocol/.document
 lib/rubygems/net-protocol/LICENSE.txt
 lib/rubygems/net-protocol/lib/net/protocol.rb
 lib/rubygems/net/http.rb
@@ -481,6 +490,7 @@ lib/rubygems/request_set/lockfile.rb
 lib/rubygems/request_set/lockfile/parser.rb
 lib/rubygems/request_set/lockfile/tokenizer.rb
 lib/rubygems/requirement.rb
+lib/rubygems/resolv/.document
 lib/rubygems/resolv/LICENSE.txt
 lib/rubygems/resolv/lib/resolv.rb
 lib/rubygems/resolver.rb
@@ -503,6 +513,7 @@ lib/rubygems/resolver/local_specification.rb
 lib/rubygems/resolver/lock_set.rb
 lib/rubygems/resolver/lock_specification.rb
 lib/rubygems/resolver/molinillo.rb
+lib/rubygems/resolver/molinillo/.document
 lib/rubygems/resolver/molinillo/LICENSE
 lib/rubygems/resolver/molinillo/lib/molinillo.rb
 lib/rubygems/resolver/molinillo/lib/molinillo/delegates/resolution_state.rb
@@ -564,6 +575,7 @@ lib/rubygems/ssl_certs/rubygems.org/GlobalSignRootCA_R3.pem
 lib/rubygems/stub_specification.rb
 lib/rubygems/text.rb
 lib/rubygems/timeout.rb
+lib/rubygems/timeout/.document
 lib/rubygems/timeout/LICENSE.txt
 lib/rubygems/timeout/lib/timeout.rb
 lib/rubygems/tsort.rb

Rakefile

@@ -105,180 +105,20 @@ RDoc::Task.new rdoc: "docs", clobber_rdoc: "clobber_docs" do |doc|
   doc.rdoc_dir = "doc"
 end
 
-# No big deal if Automatiek is not available. This might be just because
-# `rake` is executed from release tarball.
-if File.exist?("tool/automatiek.rake")
-  load "tool/automatiek.rake"
-
-  # We currently ship Molinillo master branch as of
-  # https://github.com/CocoaPods/Molinillo/commit/7cc27a355e861bdf593e2cde7bf1bca3daae4303
-  desc "Vendor a specific version of molinillo to rubygems"
-  Automatiek::RakeTask.new("molinillo") do |lib|
-    lib.version = "master"
-    lib.download = { github: "https://github.com/CocoaPods/Molinillo" }
-    lib.namespace = "Molinillo"
-    lib.prefix = "Gem::Resolver"
-    lib.vendor_lib = "lib/rubygems/resolver/molinillo"
-    lib.license_path = "LICENSE"
-
-    lib.dependency("tsort") do |sublib|
-      sublib.version = "v0.2.0"
-      sublib.download = { github: "https://github.com/ruby/tsort" }
-      sublib.namespace = "TSort"
-      sublib.prefix = "Gem"
-      sublib.vendor_lib = "lib/rubygems/tsort"
-      sublib.license_path = "LICENSE.txt"
-    end
+namespace :vendor do
+  task :bundle do
+    sh({ "BUNDLE_PATH" => "../../tmp/vendor", "BUNDLER_GEM_DEFAULT_DIR" => "../../tmp/vendor" }, "ruby", "--disable-gems", "-r./bundler/spec/support/hax.rb", "-I", "lib", "bundler/spec/support/bundle.rb", "install", "--gemfile=tool/bundler/vendor_gems.rb")
   end
 
-  # We currently ship optparse 0.3.0 plus the following changes:
-  # * Remove top aliasing the `::OptParse` constant to `OptionParser`, since we
-  #   don't need it and it triggers redefinition warnings since the default
-  #   optparse gem also does the aliasing.
-  # * Add an empty .document file to the library's root path to hint RDoc that
-  #   this library should not be documented.
-  desc "Vendor a specific version of optparse to rubygems"
-  Automatiek::RakeTask.new("optparse") do |lib|
-    lib.version = "v0.4.0"
-    lib.download = { github: "https://github.com/ruby/optparse" }
-    lib.namespace = "OptionParser"
-    lib.prefix = "Gem"
-    lib.vendor_lib = "lib/rubygems/optparse"
-    lib.license_path = "COPYING"
-  end
-
-  desc "Vendor a specific version of pub_grub to bundler"
-  Automatiek::RakeTask.new("pub_grub") do |lib|
-    lib.version = "main"
-    lib.download = { github: "https://github.com/jhawthorn/pub_grub" }
-    lib.namespace = "PubGrub"
-    lib.prefix = "Bundler"
-    lib.vendor_lib = "bundler/lib/bundler/vendor/pub_grub"
-    lib.license_path = "LICENSE.txt"
-  end
-
-  desc "Vendor a specific version of tsort to bundler"
-  Automatiek::RakeTask.new("tsort") do |lib|
-    lib.version = "v0.2.0"
-    lib.download = { github: "https://github.com/ruby/tsort" }
-    lib.namespace = "TSort"
-    lib.prefix = "Bundler"
-    lib.vendor_lib = "bundler/lib/bundler/vendor/tsort"
-    lib.license_path = "LICENSE.txt"
-  end
-
-  desc "Vendor a specific version of thor to bundler"
-  Automatiek::RakeTask.new("thor") do |lib|
-    lib.version = "v1.3.0"
-    lib.download = { github: "https://github.com/rails/thor" }
-    lib.namespace = "Thor"
-    lib.prefix = "Bundler"
-    lib.vendor_lib = "bundler/lib/bundler/vendor/thor"
-    lib.license_path = "LICENSE.md"
-  end
-
-  desc "Vendor a specific version of fileutils to bundler"
-  Automatiek::RakeTask.new("fileutils") do |lib|
-    lib.version = "v1.7.2"
-    lib.download = { github: "https://github.com/ruby/fileutils" }
-    lib.namespace = "FileUtils"
-    lib.prefix = "Bundler"
-    lib.vendor_lib = "bundler/lib/bundler/vendor/fileutils"
-    lib.license_path = "LICENSE.txt"
-  end
-
-  # We currently include the following changes over the official version:
-  # * Avoid requiring the optional `net-http-pipeline` dependency, so that its version can be selected by end users.
-  # * Require vendored net/http version RubyGems if available, otherwise the stdlib version.
-  desc "Vendor a specific version of net-http-persistent to bundler"
-  Automatiek::RakeTask.new("net-http-persistent") do |lib|
-    lib.version = "v4.0.2"
-    lib.download = { github: "https://github.com/drbrain/net-http-persistent" }
-    lib.namespace = "Net::HTTP::Persistent"
-    lib.prefix = "Gem"
-    lib.vendor_lib = "bundler/lib/bundler/vendor/net-http-persistent"
-    lib.license_path = "README.rdoc"
-
-    lib.dependency("connection_pool") do |sublib|
-      sublib.version = "v2.4.1"
-      sublib.download = { github: "https://github.com/mperham/connection_pool" }
-      sublib.namespace = "ConnectionPool"
-      sublib.prefix = "Bundler"
-      sublib.vendor_lib = "bundler/lib/bundler/vendor/connection_pool"
-      sublib.license_path = "LICENSE"
-
-      sublib.dependency("timeout") do |subsublib|
-        subsublib.version = "v0.4.1"
-        subsublib.download = { github: "https://github.com/ruby/timeout" }
-        subsublib.namespace = "Timeout"
-        subsublib.prefix = "Gem"
-        subsublib.vendor_lib = "lib/rubygems/timeout"
-        subsublib.license_path = "LICENSE.txt"
-      end
-    end
-
-    lib.dependency("uri") do |sublib|
-      sublib.version = "v0.13.0"
-      sublib.download = { github: "https://github.com/ruby/uri" }
-      sublib.namespace = "URI"
-      sublib.prefix = "Bundler"
-      sublib.vendor_lib = "bundler/lib/bundler/vendor/uri"
-      sublib.license_path = "LICENSE.txt"
-    end
-
-    lib.dependency("net-http") do |sublib|
-      sublib.version = "v0.4.0"
-      sublib.download = { github: "https://github.com/ruby/net-http" }
-      sublib.namespace = "Net"
-      sublib.prefix = "Gem"
-      sublib.vendor_lib = "lib/rubygems/net-http"
-      sublib.license_path = "LICENSE.txt"
-
-      sublib.dependency("net-protocol") do |subsublib|
-        subsublib.version = "v0.2.2"
-        subsublib.download = { github: "https://github.com/ruby/net-protocol" }
-        subsublib.namespace = "Net"
-        subsublib.prefix = "Gem"
-        subsublib.vendor_lib = "lib/rubygems/net-protocol"
-        subsublib.license_path = "LICENSE.txt"
-
-        subsublib.dependency("timeout") do |ssslib|
-          ssslib.version = "v0.4.1"
-          ssslib.download = { github: "https://github.com/ruby/timeout" }
-          ssslib.namespace = "Timeout"
-          ssslib.prefix = "Gem"
-          ssslib.vendor_lib = "lib/rubygems/timeout"
-          ssslib.license_path = "LICENSE.txt"
-        end
-      end
-
-      sublib.dependency("timeout") do |subsublib|
-        subsublib.version = "v0.4.1"
-        subsublib.download = { github: "https://github.com/ruby/timeout" }
-        subsublib.namespace = "Timeout"
-        subsublib.prefix = "Gem"
-        subsublib.vendor_lib = "lib/rubygems/timeout"
-        subsublib.license_path = "LICENSE.txt"
-      end
+  task install: :bundle do
+    sh({ "BUNDLE_GEMFILE" => "tool/bundler/vendor_gems.rb", "BUNDLE_PATH" => "../../tmp/vendor", "BUNDLER_GEM_DEFAULT_DIR" => "../../tmp/vendor" }, "ruby", "-rpathname", "-r./bundler/spec/support/hax.rb", "-I", "lib", "bundler/spec/support/bundle.rb", "exec", "tool/automatiek/vendor.rb")
+  end
 
-      sublib.dependency("resolv") do |subsublib|
-        subsublib.version = "v0.3.0"
-        subsublib.download = { github: "https://github.com/ruby/resolv" }
-        subsublib.namespace = "Resolv"
-        subsublib.prefix = "Gem"
-        subsublib.vendor_lib = "lib/rubygems/resolv"
-        subsublib.license_path = "LICENSE.txt"
-
-        subsublib.dependency("timeout") do |ssslib|
-          ssslib.version = "v0.4.1"
-          ssslib.download = { github: "https://github.com/ruby/timeout" }
-          ssslib.namespace = "Timeout"
-          ssslib.prefix = "Gem"
-          ssslib.vendor_lib = "lib/rubygems/timeout"
-          ssslib.license_path = "LICENSE.txt"
-        end
-      end
-    end
+  task check: :install do
+    Spec::Rubygems.check_source_control_changes(
+      success_message: "Vendored gems are in sync",
+      error_message: "Vendored gems are out of sync. Please update the vendored lib patches."
+    )
   end
 end
 

bundler/lib/bundler/vendor/connection_pool/.document

@@ -0,0 +1 @@
+# Vendored files do not need to be documented

bundler/lib/bundler/vendor/fileutils/.document

@@ -0,0 +1 @@
+# Vendored files do not need to be documented

bundler/lib/bundler/vendor/net-http-persistent/.document

@@ -0,0 +1 @@
+# Vendored files do not need to be documented

bundler/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent.rb

@@ -20,7 +20,7 @@
 #
 # Example:
 #
-#   require 'bundler/vendor/net-http/lib/net/http/persistent'
+#   require 'bundler/vendor/net-http-persistent/lib/net/http/persistent'
 #
 #   uri = Bundler::URI 'http://example.com/awesome/web/service'
 #

bundler/lib/bundler/vendor/pub_grub/.document

@@ -0,0 +1 @@
+# Vendored files do not need to be documented

bundler/lib/bundler/vendor/thor/.document

@@ -0,0 +1 @@
+# Vendored files do not need to be documented

bundler/lib/bundler/vendor/tsort/.document

@@ -0,0 +1 @@
+# Vendored files do not need to be documented

bundler/lib/bundler/vendor/uri/.document

@@ -0,0 +1 @@
+# Vendored files do not need to be documented

lib/rubygems/net-http/.document

@@ -0,0 +1 @@
+# Vendored files do not need to be documented

lib/rubygems/net-protocol/.document

@@ -0,0 +1 @@
+# Vendored files do not need to be documented

lib/rubygems/resolv/.document

@@ -0,0 +1 @@
+# Vendored files do not need to be documented

lib/rubygems/resolver/molinillo/.document

@@ -0,0 +1 @@
+# Vendored files do not need to be documented

lib/rubygems/timeout/.document

@@ -0,0 +1 @@
+# Vendored files do not need to be documented

tool/automatiek/connection_pool-v2.4.1.patch

@@ -0,0 +1,10 @@
+diff --git b/bundler/lib/bundler/vendor/connection_pool/lib/connection_pool.rb a/bundler/lib/bundler/vendor/connection_pool/lib/connection_pool.rb
+index e680ab363..317088a86 100644
+--- b/bundler/lib/bundler/vendor/connection_pool/lib/connection_pool.rb
++++ a/bundler/lib/bundler/vendor/connection_pool/lib/connection_pool.rb
+@@ -1,4 +1,4 @@
+-require_relative "../../../../../../lib/rubygems/timeout/lib/timeout"
++require_relative "../../../vendored_timeout"
+ require_relative "connection_pool/version"
+
+ class Bundler::ConnectionPool