Permalink
Browse files

Full run, with warn every 50

  • Loading branch information...
tmtmtmtm committed Sep 12, 2015
1 parent a0f1eb0 commit 1c7e253c4e0fc0fb41df59a5bd5736a58d0d612d
Showing with 4 additions and 5 deletions.
  1. +4 −5 scraper.rb
View
@@ -108,15 +108,14 @@ def scrape_person(i)
data = person.merge(t)
data[:party] = data[:party].sub('Parliamentary Group of ','').sub('Independent Members of Parliament', 'Independent')
data[:term] = data[:term][/^(\d+)/, 1] rescue binding.pry
puts "%s = %s" % [data[:id], data[:party]]
ScraperWiki.save_sqlite([:id, :term, :party, :start_date], data)
end
end
rows = ScraperWiki.select("DISTINCT(id) from 'data' WHERE party LIKE 'Parliamentary Group%'")
rows.map { |r| r['id'] }.each do |i|
# (1..2650).each do |i|
puts "Deleting #{i}"
# rows = ScraperWiki.select("DISTINCT(id) from 'data' WHERE party LIKE 'Parliamentary Group%'")
# rows.map { |r| r['id'] }.each do |i|
(1..2650).each do |i|
warn i if (i % 50).zero?
ScraperWiki.sqliteexecute('DELETE FROM data WHERE id = ?', i)
scrape_person(i)
end

0 comments on commit 1c7e253

Please sign in to comment.