Add job to upload pre-rendered versions files to S3

app/controllers/api/compact_index_controller.rb

@@ -29,7 +29,7 @@ def info
 
   def render_range(response_body)
     headers["ETag"] = '"' << Digest::MD5.hexdigest(response_body) << '"'
-    headers["Digest"] = "sha-256=#{Base64.encode64(Digest::SHA256.digest(response_body)).strip}"
+    headers["Digest"] = "sha-256=#{Digest::SHA256.base64digest(response_body)}"
     headers["Accept-Ranges"] = "bytes"
 
     ranges = Rack::Utils.byte_ranges(request.env, response_body.bytesize)

app/jobs/upload_versions_file_job.rb

@@ -0,0 +1,42 @@
+class UploadVersionsFileJob < ApplicationJob
+  queue_with_priority PRIORITIES.fetch(:push)
+
+  include GoodJob::ActiveJobExtensions::Concurrency
+  good_job_control_concurrency_with(
+    # Maximum number of jobs with the concurrency key to be
+    # concurrently enqueued (excludes performing jobs)
+    #
+    # Because the job only uses current state at time of perform,
+    # it makes no sense to enqueue more than one at a time
+    enqueue_limit: good_job_concurrency_enqueue_limit(default: 1),
+    perform_limit: good_job_concurrency_perform_limit(default: 1),
+    key: name
+  )
+
+  def perform
+    versions_path = Rails.application.config.rubygems["versions_file_location"]
+    versions_file = CompactIndex::VersionsFile.new(versions_path)
+    from_date = versions_file.updated_at
+
+    logger.info "Generating versions file from #{from_date}"
+
+    extra_gems = GemInfo.compact_index_versions(from_date)
+    response_body = CompactIndex.versions(versions_file, extra_gems)
+
+    md5 = Digest::MD5.new.update(response_body)
+    checksum_sha256 = Digest::SHA256.base64digest(response_body)
+
+    response = RubygemFs.compact_index.store(
+      "versions", response_body,
+      metadata: { "surrogate-key" => "versions s3-compact-index s3-versions" },
+      cache_control: "max-age=60, public",
+      content_type: "text/plain; charset=utf-8",
+      checksum_sha256:,
+      content_md5: md5.base64digest
+    )
+
+    logger.info(message: "Uploading versions file succeeded", response:)
+
+    FastlyPurgeJob.perform_later(key: "s3-versions", soft: true)
+  end
+end

app/models/pusher.rb

@@ -128,6 +128,7 @@ def after_write
       Mailer.gem_pushed(user.id, @version_id, notified_user.id).deliver_later
     end
     Indexer.perform_later
+    UploadVersionsFileJob.perform_later
     ReindexRubygemJob.perform_later(rubygem:)
     GemCachePurger.call(rubygem.name)
     StoreVersionContentsJob.perform_later(version:) if ld_variation(key: "gemcutter.pusher.store_version_contents", default: false)
@@ -159,6 +160,7 @@ def update
   end
 
   def set_info_checksum
+    # TODO: may as well just upload this straight to S3...
     checksum = GemInfo.new(rubygem.name).info_checksum
     version.update_attribute :info_checksum, checksum
   end

config/rubygems.yml

@@ -3,6 +3,7 @@ development:
   host: localhost
   s3_bucket: s3bucket
   s3_contents_bucket: contents
+  s3_compact_index_bucket: compact-index
   s3_region: us-east-1
   s3_endpoint: s3.amazonaws.com
   versions_file_location: "./config/versions.list"
@@ -12,6 +13,7 @@ test:
   host: localhost
   s3_bucket: test.s3.rubygems.org
   s3_contents_bucket: contents.test.s3.rubygems.org
+  s3_compact_index_bucket: compact-index.test.s3.rubygems.org
   s3_region: us-east-1
   s3_endpoint: s3.amazonaws.com
   versions_file_location: "./test/helpers/versions.list"
@@ -23,6 +25,7 @@ staging:
   s3_region: us-west-2
   s3_endpoint: s3-us-west-2.amazonaws.com
   s3_contents_bucket: contents.oregon.staging.s3.rubygems.org
+  s3_compact_index_bucket: compact-index.oregon.staging.s3.rubygems.org
   versions_file_location: "./config/versions.list"
 
 production:
@@ -32,6 +35,7 @@ production:
   s3_region: us-west-2
   s3_endpoint: s3-us-west-2.amazonaws.com
   s3_contents_bucket: contents.oregon.production.s3.rubygems.org
+  s3_compact_index_bucket: compact-index.oregon.production.s3.rubygems.org
   versions_file_location: "./config/versions.list"
   separate_admin_host: rubygems.team
 

lib/rubygem_fs.rb

@@ -16,13 +16,19 @@ def self.contents
     @contents ||= instance.in_bucket Gemcutter.config.s3_contents_bucket
   end
 
+  def self.compact_index
+    @compact_index ||= instance.in_bucket Gemcutter.config.s3_compact_index_bucket
+  end
+
   def self.mock!
     @contents = nil
+    @compact_index = nil
     @fs = RubygemFs::Local.new(Dir.mktmpdir)
   end
 
   def self.s3!(host)
     @contents = nil
+    @compact_index = nil
     @fs = RubygemFs::S3.new(access_key_id: "k",
                             secret_access_key: "s",
                             endpoint: host,
@@ -155,7 +161,7 @@ def in_bucket(bucket)
     end
 
     def store(key, body, metadata: {}, **kwargs)
-      allowed_args = kwargs.slice(:content_type, :checksum_sha256, :content_encoding)
+      allowed_args = kwargs.slice(:content_type, :checksum_sha256, :content_encoding, :cache_control, :content_md5)
       s3.put_object(key: key,
                     body: body,
                     bucket: bucket,

test/jobs/upload_versions_file_job_test.rb

@@ -0,0 +1,34 @@
+require "test_helper"
+
+class UploadVersionsFileJobTest < ActiveJob::TestCase
+  test "uploads the versions file" do
+    version = create(:version)
+    info_checksum = GemInfo.new(version.rubygem.name).info_checksum
+    version.update!(info_checksum:)
+
+    UploadVersionsFileJob.perform_now
+
+    content = <<~VERSIONS
+      created_at: 2015-08-23T17:22:53-07:00
+      ---
+      old_gem_one 1 e63fe62df0f1f459d2f70986f79745c6
+      old_gem_two 0.1.0,0.1.1,0.1.2,0.2.0,0.2.1,0.3.0,0.4.0,0.4.1,0.5.0,0.5.1,0.5.2,0.5.3 c54e4b7e14861a5d8c225283b75075f4
+      #{version.rubygem.name} #{version.number} #{info_checksum}
+    VERSIONS
+
+    assert_equal content, RubygemFs.compact_index.get("versions")
+
+    assert_equal(
+      {
+        metadata: { "surrogate-key" => "versions s3-compact-index s3-versions" },
+        cache_control: "max-age=60, public",
+        content_type: "text/plain; charset=utf-8",
+        checksum_sha256: Digest::SHA256.base64digest(content),
+        content_md5: Digest::MD5.base64digest(content),
+        key: "versions"
+      }, RubygemFs.compact_index.head("versions")
+    )
+
+    assert_enqueued_with(job: FastlyPurgeJob, args: [{ key: "s3-versions", soft: true }])
+  end
+end