use a more compact, per-shard index for all the pods

lib/cocoapods-core/cdn_source.rb

@@ -20,6 +20,9 @@ def initialize(repo)
         :max_threads => 200,
         :max_queue => 0 # unbounded work queue
       )
+
+      @version_arrays_by_fragment_by_name = {}
+
       super(repo)
     end
 
@@ -84,36 +87,34 @@ def versions(name)
       return nil unless specs_dir
       raise ArgumentError, 'No name' unless name
 
+      fragment = pod_shard_fragment(name)
+
+      ensure_versions_file_loaded(fragment)
+
       return @versions_by_name[name] unless @versions_by_name[name].nil?
 
       pod_path_actual = pod_path(name)
       pod_path_relative = relative_pod_path(name)
-      versions_file_path_relative = pod_path_relative.join(INDEX_FILE_NAME).to_s
-      download_file(versions_file_path_relative)
 
-      return nil unless pod_path_actual.join(INDEX_FILE_NAME).exist?
+      return nil if @version_arrays_by_fragment_by_name[fragment][name].nil?
 
       loaders = []
-      @versions_by_name[name] ||= local_file(versions_file_path_relative) do |file|
-        file.map do |v|
-          version = v.chomp
-
-          # Optimization: ensure all the podspec files at least exist. The correct one will get refreshed
-          # in #specification_path regardless.
-          podspec_version_path_relative = Pathname.new(version).join("#{name}.podspec.json")
-          unless pod_path_actual.join(podspec_version_path_relative).exist?
-            loaders << Concurrent::Promise.execute(:executor => @executor) do
-              download_file(pod_path_relative.join(podspec_version_path_relative).to_s)
-            end
-          end
-          begin
-            Version.new(version) if version[0, 1] != '.'
-          rescue ArgumentError
-            raise Informative, 'An unexpected version directory ' \
-            "`#{version}` was encountered for the " \
-            "`#{pod_dir}` Pod in the `#{name}` repository."
+      @versions_by_name[name] ||= @version_arrays_by_fragment_by_name[fragment][name].map do |version|
+        # Optimization: ensure all the podspec files at least exist. The correct one will get refreshed
+        # in #specification_path regardless.
+        podspec_version_path_relative = Pathname.new(version).join("#{name}.podspec.json")
+        unless pod_path_actual.join(podspec_version_path_relative).exist?
+          loaders << Concurrent::Promise.execute(:executor => @executor) do
+            download_file(pod_path_relative.join(podspec_version_path_relative).to_s)
           end
         end
+        begin
+          Version.new(version) if version[0, 1] != '.'
+        rescue ArgumentError
+          raise Informative, 'An unexpected version directory ' \
+          "`#{version}` was encountered for the " \
+          "`#{pod_dir}` Pod in the `#{name}` repository."
+        end
       end.compact.sort.reverse
       Concurrent::Promise.zip(*loaders).wait!
       @versions_by_name[name]
@@ -146,6 +147,12 @@ def all_specs
       raise Informative, "Can't retrieve all the specs for a CDN-backed source, it will take forever"
     end
 
+    # @return [Array<Sets>] the sets of all the Pods.
+    #
+    def pod_sets
+      raise Informative, "Can't retrieve all the pod sets for a CDN-backed source, it will take forever"
+    end
+
     # @!group Searching the source
     #-------------------------------------------------------------------------#
 
@@ -165,7 +172,13 @@ def search(query)
         query = query.root_name
       end
 
-      found = download_file(relative_pod_path(query).join(INDEX_FILE_NAME).to_s)
+      fragment = pod_shard_fragment(query)
+
+      ensure_versions_file_loaded(fragment)
+
+      version_arrays_by_name = @version_arrays_by_fragment_by_name[fragment] || {}
+
+      found = version_arrays_by_name[query].nil? ? nil : query
 
       if found
         set = set(query)
@@ -220,12 +233,37 @@ def git?
 
     private
 
-    # Index files contain all the sub directories in the directory, separated by
-    # a newline. We use those because you can't get a directory listing from a CDN.
-    INDEX_FILE_NAME = 'index.txt'.freeze
+    def ensure_versions_file_loaded(fragment)
+      return if !@version_arrays_by_fragment_by_name[fragment].nil? && !@check_existing_files_for_update
+
+      # Index file that contains all the versions for all the pods in the shard.
+      # We use those because you can't get a directory listing from a CDN.
+      index_file_name = index_file_name_for_fragment(fragment)
+      download_file(index_file_name)
+      versions_raw = local_file(index_file_name, &:to_a).map(&:chomp)
+      @version_arrays_by_fragment_by_name[fragment] = versions_raw.reduce({}) do |hash, row|
+        row = row.split('/')
+        pod = row.shift
+        versions = row
+
+        hash[pod] = versions
+        hash
+      end
+    end
+
+    def index_file_name_for_fragment(fragment)
+      fragment_joined = fragment.join('_')
+      fragment_joined = '_' + fragment_joined unless fragment.empty?
+      "all_pods_versions#{fragment_joined}.txt"
+    end
+
+    def pod_shard_fragment(pod_name)
+      metadata.path_fragment(pod_name)[0..-2]
+    end
 
     def local_file(partial_url)
-      File.open(repo.join(partial_url)) do |file|
+      file_path = repo.join(partial_url)
+      File.open(file_path) do |file|
         yield file if block_given?
       end
     end
@@ -257,9 +295,18 @@ def download_file(partial_url)
       etag = File.read(etag_path) if File.exist?(etag_path)
       debug "CDN: #{name} Relative path: #{partial_url}, has ETag? #{etag}" unless etag.nil?
 
+      download_from_url(partial_url, file_remote_url, etag)
+    end
+
+    def download_from_url(partial_url, file_remote_url, etag)
+      path = repo + partial_url
+      etag_path = path.sub_ext(path.extname + '.etag')
+
       response = etag.nil? ? REST.get(file_remote_url) : REST.get(file_remote_url, 'If-None-Match' => etag)
 
       case response.status_code
+      when 301
+        download_from_url(partial_url, response.headers['location'].first, etag)
       when 304
         debug "CDN: #{name} Relative path not modified: #{partial_url}"
         # We need to update the file modification date, as it is later used for freshness

lib/cocoapods-core/source/metadata.rb

@@ -47,7 +47,7 @@ def path_fragment(pod_name, version = nil)
                        hashed.slice!(0, length)
                      end
                    end
-        prefixes.concat([pod_name, version]).compact.join(File::SEPARATOR)
+        prefixes.concat([pod_name, version]).compact
       end
 
       def last_compatible_version(target_version)

spec/cdn_source_spec.rb

@@ -4,9 +4,26 @@
 module Pod
   describe CDNSource do
     before do
+      def get_canonical_versions(relative_path)
+        path = @remote_dir.join(relative_path)
+        File.read(path).split("\n")
+      end
+
+      def get_etag(relative_path)
+        path = @source.repo.join(relative_path)
+        etag_path = path.sub_ext(path.extname + '.etag')
+        File.read(etag_path) if File.exist?(etag_path)
+      end
+
+      def all_local_files
+        [@source.repo.join('**/*.yml'), @source.repo.join('**/*.txt'), @source.repo.join('**/*.json')].map(&Pathname.method(:glob)).flatten
+      end
+
       @path = fixture('spec-repos/test_cdn_repo_local')
       Pathname.glob(@path.join('*')).each(&:rmtree)
       @source = CDNSource.new(@path)
+
+      @remote_dir = fixture('mock_cdn_repo_remote')
     end
 
     after do
@@ -48,7 +65,6 @@ module Pod
       end
 
       it 'returns nil if the Pod could not be found' do
-        @source.expects(:debug).with { |cmd| cmd =~ %r{CDN: #{@source.name} Relative path couldn't be downloaded: Specs\/.*\/Unknown_Pod\/index\.txt Response: 404} }
         @source.versions('Unknown_Pod').should.be.nil
       end
 
@@ -102,9 +118,10 @@ module Pod
     #-------------------------------------------------------------------------#
 
     describe '#pod_sets' do
-      it 'returns all the pod sets' do
-        expected = %w(BeaconKit SDWebImage)
-        @source.pod_sets.map(&:name).sort.uniq.should == expected
+      it 'raises an error' do
+        should.raise Informative do
+          @source.pod_sets
+        end.message.should.match /Can't retrieve all the pod sets for a CDN-backed source, it will take forever/
       end
     end
 
@@ -126,7 +143,7 @@ module Pod
       end
 
       it 'matches case' do
-        @source.expects(:debug).with { |cmd| cmd =~ %r{CDN: #{@source.name} Relative path couldn't be downloaded: Specs\/.*\/bEacoNKIT\/index\.txt Response: 404} }
+        @source.expects(:debug).with { |cmd| cmd =~ /CDN: #{@source.name} Relative path downloaded: all_pods_versions_9_5_b\.txt, save ETag:/ }
         @source.search('bEacoNKIT').should.be.nil?
       end
 
@@ -219,30 +236,23 @@ module Pod
         @source.search('BeaconKit')
       end
 
-      def get_versions(relative_path)
-        path = @source.repo.join(relative_path)
-        File.read(path).split("\n")
-      end
-
       it 'refreshes all index files' do
         @source.expects(:download_file).with('CocoaPods-version.yml').returns('CocoaPods-version.yml')
-        pod_relative_path = @source.pod_path('BeaconKit').relative_path_from(@source.repo).join('index.txt').to_s
-        @source.expects(:download_file).with(pod_relative_path).returns(pod_relative_path)
-        get_versions(pod_relative_path).each do |version|
-          podspec_relative_path = @source.pod_path('BeaconKit').relative_path_from(@source.repo).join(version).join('BeaconKit.podspec.json').to_s
-          @source.expects(:download_file).with(podspec_relative_path).returns(podspec_relative_path)
-        end
-        @source.update(true)
-      end
+        @source.expects(:download_file).with('all_pods_versions_2_0_9.txt').returns('all_pods_versions_2_0_9.txt')
 
-      def get_etag(relative_path)
-        path = @source.repo.join(relative_path)
-        etag_path = path.sub_ext(path.extname + '.etag')
-        File.read(etag_path) if File.exist?(etag_path)
-      end
+        ['BeaconKit/1.0.0/1.0.1/1.0.2/1.0.3/1.0.4/1.0.5', 'SDWebImage/2.4/2.5/2.6/2.7/2.7.4/3.0/3.1/4.0.0/4.0.0-beta/4.0.0-beta2'].each do |row|
+          row = row.split('/')
+          pod = row.shift
+          versions = row
 
-      def all_local_files
-        [@source.repo.join('**/*.yml'), @source.repo.join('**/*.txt'), @source.repo.join('**/*.json')].map(&Pathname.method(:glob)).flatten
+          next unless pod == 'BeaconKit'
+
+          versions.each do |version|
+            podspec_relative_path = @source.pod_path(pod).relative_path_from(@source.repo).join(version).join("#{pod}.podspec.json").to_s
+            @source.expects(:download_file).with(podspec_relative_path).returns(podspec_relative_path)
+          end
+        end
+        @source.update(true)
       end
 
       it 'handles ETag and If-None-Match headers' do

spec/fixtures/mock_cdn_repo_remote/all_pods_versions_1_1_7.txt

@@ -0,0 +1 @@
+SDWebImage/2.4/2.5/2.6/2.7/2.7.4/3.0/3.1/4.0.0/4.0.0-beta/4.0.0-beta2

spec/fixtures/mock_cdn_repo_remote/all_pods_versions_2_0_9.txt

@@ -0,0 +1 @@
+BeaconKit/1.0.0/1.0.1/1.0.2/1.0.3/1.0.4/1.0.5