fetch in batches of 100

everypolitician-scrapers · Mar 25, 2016 · c41e4ed · c41e4ed
1 parent cf7358b
commit c41e4ed
Showing 1 changed file with 1 addition and 4 deletions.
diff --git a/scraper.rb b/scraper.rb
@@ -27,8 +27,5 @@
 ]
 
 names = @pages.map { |p| WikiData::Category.new( p, 'en').member_titles }.flatten.uniq
-
-names.shuffle.each_slice(50) do |sliced|
-  EveryPolitician::Wikidata.scrape_wikidata(names: { he: [], en: sliced })
-end
+EveryPolitician::Wikidata.scrape_wikidata(names: { he: [], en: names }, batch_size: 50)