Fix crawler to skip duplicated items correctly

lib/fastladder/crawler.rb

@@ -180,7 +180,7 @@ def cut_off(items)
     end
 
     def reject_duplicated(feed, items)
-      items.reject { |item| feed.items.exists?(["guid = ? and digest = ?", item.id, item.digest]) }
+      items.uniq { |item| item.guid }.reject { |item| feed.items.exists?(["guid = ? and digest = ?", item.guid, item.digest]) }
     end
 
     def delete_old_items_if_new_items_are_many(new_items_size)

spec/factories/items.rb

@@ -29,4 +29,8 @@
   factory :item_without_guid, parent: :item do
     guid nil
   end
+
+  factory :item_has_fixed_guid, parent: :item do
+    guid "guid"
+  end
 end

spec/lib/fastladder/crawler_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+
+describe 'Fastladder::Crawler' do
+  let(:crawler) { Fastladder::Crawler.new(Rails.logger) }
+  let(:feed) { FactoryGirl.create(:feed) }
+
+  context 'when some items have same guid' do
+    let(:items) { FactoryGirl.build_list(:item_has_fixed_guid, 2) }
+
+    describe '#reject_duplicated' do
+      it 'takes the first of them' do
+        expect(crawler.send(:reject_duplicated, feed, items)).to eq(items.take(1))
+      end
+    end
+  end
+
+  context 'when items are duplicated' do
+    let(:items) { FactoryGirl.build_list(:item_has_fixed_guid, 1) }
+    before {
+      FactoryGirl.create(:item_has_fixed_guid, feed: feed)
+      items.each { |item| item.create_digest }
+    }
+
+    describe '#reject_duplicated' do
+      it 'rejects them' do
+        expect(crawler.send(:reject_duplicated, feed, items)).to be_empty
+      end
+    end
+  end
+end