Skip to content

Commit

Permalink
Index on Statement, not Person
Browse files Browse the repository at this point in the history
This is guaranteed to be unique
  • Loading branch information
tmtmtmtm committed Apr 25, 2019
1 parent 35715de commit e301e70
Showing 1 changed file with 6 additions and 2 deletions.
8 changes: 6 additions & 2 deletions scraper.rb
Expand Up @@ -14,6 +14,10 @@ class Results < Scraped::JSON
end

class Membership < Scraped::JSON
field :statement do
json.dig(:ps, :value).to_s.split('/').last
end

field :id do
json.dig(:item, :value).to_s.split('/').last
end
Expand Down Expand Up @@ -54,7 +58,7 @@ class Membership < Scraped::JSON
WIKIDATA_SPARQL_URL = 'https://query.wikidata.org/sparql?format=json&query=%s'

memberships_query = <<SPARQL
SELECT ?item ?itemLabel ?group ?groupLabel ?district ?districtLabel ?start ?end ?term ?termLabel ?termOrdinal
SELECT ?ps ?item ?itemLabel ?group ?groupLabel ?district ?districtLabel ?start ?end ?term ?termLabel ?termOrdinal
{
?item p:P39 ?ps .
?ps ps:P39/wdt:P279* wd:Q3406079 .
Expand All @@ -73,4 +77,4 @@ class Membership < Scraped::JSON
puts data.map(&:compact).map(&:sort).map(&:to_h) if ENV['MORPH_DEBUG']

ScraperWiki.sqliteexecute('DROP TABLE data') rescue nil
ScraperWiki.save_sqlite(%i[id term party_id start_date], data)
ScraperWiki.save_sqlite(%i[statement], data)

0 comments on commit e301e70

Please sign in to comment.