Add support for yanked gems on compact index

app/models/deletion.rb

@@ -33,7 +33,7 @@ def expire_api_memcached
   end
 
   def remove_from_index
-    @version.update!(indexed: false)
+    @version.update!(indexed: false, yanked_at: Time.now.utc)
     Delayed::Job.enqueue Indexer.new, priority: PRIORITIES[:push]
   end
 

app/models/rubygem.rb

@@ -262,10 +262,20 @@ def first_built_date
   end
 
   def self.compact_index_versions(date)
-    gems = Rubygem.joins(:versions)
-      .select("name, versions.created_at, number, platform, info_checksum")
-      .where("indexed = true and versions.created_at > ?", date)
-      .order("created_at, versions.number, platform")
+    created_gems = Rubygem.joins(:versions)
+      .select("name, versions.created_at as date, number, platform, info_checksum")
+      .where("versions.created_at > ?", date)
+      .order("versions.created_at, versions.number, platform")
+
+    yanked_gems = Rubygem.joins(:versions)
+      .select("name, versions.yanked_at as date, '-'||number, platform, info_checksum")
+      .where("indexed is false and versions.created_at > ?", date)
+      .order("versions.created_at, versions.number, platform")
+
+    # Arel don't support yet order on unions :(
+    # https://github.com/rails/arel/issues/98
+    sql = "(#{created_gems.to_sql}) UNION (#{yanked_gems.to_sql}) ORDER BY date"
+    gems = Rubygem.find_by_sql(sql)
 
     gems.map do |gem|
       CompactIndex::Gem.new(gem.name, [

db/migrate/20151123153040_add_yanked_at_to_versions.rb

@@ -0,0 +1,5 @@
+class AddYankedAtToVersions < ActiveRecord::Migration
+  def change
+    add_column :versions, :yanked_at, :datetime
+  end
+end

db/schema.rb

@@ -214,6 +214,7 @@
     t.hstore   "metadata",          default: {},   null: false
     t.string   "required_rubygems_version"
     t.string   "info_checksum"
+    t.datetime "yanked_at"
   end
 
   add_index "versions", ["built_at"], name: "index_versions_on_built_at", using: :btree

test/unit/deletion_test.rb

@@ -34,6 +34,10 @@ class DeletionTest < ActiveSupport::TestCase
       refute @version.reload.latest?
     end
 
+    should "keep the yanked time" do
+      assert @version.reload.yanked_at
+    end
+
     should "delete the .gem file" do
       assert_nil RubygemFs.instance.get("gems/#{@version.full_name}.gem"), "Rubygem still exists!"
     end