Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Also generate list of cabinet positions
  • Loading branch information
tmtmtmtm committed Mar 4, 2019
1 parent 8a4a572 commit 767ee74
Show file tree
Hide file tree
Showing 3 changed files with 58 additions and 52 deletions.
60 changes: 8 additions & 52 deletions .rubocop.yml
@@ -1,52 +1,8 @@
Style/AlignHash:
EnforcedHashRocketStyle: table
EnforcedColonStyle: table

Style/ClassAndModuleCamelCase:
Enabled: false

Style/CollectionMethods:
Enabled: true

Style/Documentation:
Enabled: false

Style/FormatString:
EnforcedStyle: percent

Style/HashSyntax:
EnforcedStyle: ruby19_no_mixed_keys

Style/RescueModifier:
Enabled: false

Style/SignalException:
Enabled: false

Style/SymbolArray:
Enabled: true

Style/TrailingCommaInLiteral:
EnforcedStyleForMultiline: consistent_comma


Metrics/ClassLength:
Max: 250

Metrics/MethodLength:
Max: 20

Metrics/LineLength:
Max: 150

Metrics/AbcSize:
Max: 50

Metrics/CyclomaticComplexity:
Max: 7

Metrics/PerceivedComplexity:
Max: 8

# Get rid of these ones over time
inherit_from: .rubocop_todo.yml
AllCops:
Exclude:
- 'Vagrantfile'
- 'vendor/**/*'
TargetRubyVersion: 2.4

inherit_from:
- https://raw.githubusercontent.com/everypolitician/everypolitician-data/master/.rubocop_base.yml
44 changes: 44 additions & 0 deletions lib/cabinet.rb
@@ -1,5 +1,8 @@
# frozen_string_literal: true

require 'cgi'
require 'csv'

# TODO: extend Scraped::Scraper with ability to add Strategies
class Scraped::Request::Strategy::LiveRequest
require 'rest-client'
Expand Down Expand Up @@ -107,3 +110,44 @@ class Membership < Scraped::JSON
end
end
end

# TODO: combine this more fully with above Strategy
class Cabinet
def initialize(cabinet:)
@cabinet = cabinet
end

def positions
sparql(position_query % cabinet).map(&:to_h).map do |r|
{
id: r[:item].to_s.split('/').last,
label: r[:itemlabel].to_s,
type: 'cabinet',
}
end
end

private

attr_reader :cabinet

WIKIDATA_SPARQL_URL = 'https://query.wikidata.org/sparql'

def sparql(query)
result = RestClient.get WIKIDATA_SPARQL_URL, accept: 'text/csv', params: { query: query }
CSV.parse(result, headers: true, header_converters: :symbol)
rescue RestClient::Exception => e
raise "Wikidata query #{query} failed: #{e.message}"
end

def position_query
<<~SPARQL
SELECT DISTINCT ?item ?itemLabel WHERE {
?item wdt:P31/wdt:P279* wd:Q4164871 ; wdt:P361 wd:%s .
SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
# date = #{Time.now}
}
ORDER BY ?item
SPARQL
end
end
6 changes: 6 additions & 0 deletions scraper.rb 100644 → 100755
Expand Up @@ -6,3 +6,9 @@
require_relative 'lib/cabinet'

Scraped::Scraper.new('Q1711695' => CabinetScraper).store(:memberships, index: %i[position_id])

positions = Cabinet.new(cabinet: 'Q32859621').positions
puts positions if ENV['MORPH_DEBUG']

ScraperWiki.sqliteexecute('DROP TABLE positions') rescue nil
ScraperWiki.save_sqlite([:id], positions, 'positions')

0 comments on commit 767ee74

Please sign in to comment.