-
Notifications
You must be signed in to change notification settings - Fork 0
/
scraper.rb
69 lines (59 loc) · 1.77 KB
/
scraper.rb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
#!/bin/env ruby
# encoding: utf-8
require 'scraperwiki'
require 'nokogiri'
require 'date'
require 'open-uri'
require 'open-uri/cached'
require 'colorize'
require 'pry'
require 'csv'
def noko(url)
Nokogiri::HTML(open(url).read)
end
@terms = {
'2012' => 'List_of_MPs_elected_in_the_Mongolian_legislative_election,_2012',
'2008' => 'List_of_MPs_elected_in_the_Mongolian_legislative_election,_2008',
}
@terms.each do |term, pagename|
url = "https://en.wikipedia.org/wiki/" + pagename
page = noko(url)
# Constituency based
members = page.xpath('.//h2/span[text()[contains(.,"Constituency")]]/following::table[1]')
# Store this outside the loop so we can refer back in rowspans
district = nil
members.xpath('.//tr[td]').each do |tr|
tds = tr.xpath('./td')
if tds.count == 5
district = tds[0]
else
# Nokogiri::XML::NodeSet doesn't have an unshift
tds = [district, tds].flatten
end
data = {
name: tds[1].xpath('.//a').text.strip,
name__mn: tds[2].text.strip,
party: tds[4].text.strip,
constituency: tds[0].text.strip.gsub("\n",' — '),
term: term,
wikiname: tds[1].xpath('.//a[not(@class="new")]/@title').text.strip,
source: url,
}
ScraperWiki.save_sqlite([:name, :term], data)
end
# Party List
partylist = page.xpath('.//h2/span[text()[contains(.,"Party list")]]/following::table[1]')
partylist.xpath('.//tr[td]').each do |tr|
tds = tr.xpath('./td')
data = {
name: tds[0].xpath('.//a').text.strip,
name__mn: tds[1].text.strip,
party: tds[3].text.strip,
constituency: 'n/a',
term: term,
wikiname: tds[1].xpath('.//a[not(@class="new")]/@title').text.strip,
source: url,
}
ScraperWiki.save_sqlite([:name, :term], data)
end
end