Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

basic person data loader

  • Loading branch information...
commit 0e959eb015f0158e95f8d75b53325d2b4f4bbd44 1 parent 2f1293f
@lizconlan authored
Showing with 48 additions and 4 deletions.
  1. +9 −4 Rakefile
  2. +27 −0 lib/person_loader.rb
  3. +12 −0 models/person.rb
View
13 Rakefile
@@ -9,6 +9,7 @@ require_relative 'lib/regnal_year_loader'
require_relative 'lib/by_election_loader'
require_relative 'lib/general_election_loader'
require_relative 'lib/election_results_loader'
+require_relative 'lib/person_loader'
MONGO_URL = ENV['MONGOHQ_URL'] || YAML::load(File.read("config/mongo.yml"))[:mongohq_url]
env = {}
@@ -23,15 +24,19 @@ task :load_all do
#
# ryl = RegnalYearLoader.new
# ryl.load_from_scraperwiki()
- #
+ #
+ per = PersonLoader.new
+ per.load_from_scraperwiki()
+ #
# bel = ByElectionLoader.new
# bel.load_from_file()
#
# gel = GeneralElectionLoader.new
# gel.load_from_file()
-
- res = GeneralElectionResultsLoader.new
- res.load_from_the_guardian()
+ #
+ #
+ # res = GeneralElectionResultsLoader.new
+ # res.load_from_the_guardian()
end
require 'rake/testtask'
View
27 lib/person_loader.rb
@@ -0,0 +1,27 @@
+require 'rest-client'
+require 'json'
+require 'date'
+require_relative '../models/person.rb'
+
+class PersonLoader
+ def load_from_scraperwiki(scraper_name="parly_people")
+ url = "https://api.scraperwiki.com/api/1.0/datastore/sqlite?format=jsondict&name=#{scraper_name}&query=select%20*%20from%20%60swdata%60"
+ response = RestClient.get(url)
+ data = JSON.parse(response.body)
+ data.each do |record|
+ person = Person.new
+ person.title = record["title"]
+ person.forenames = record["forenames"]
+ person.surname = record["surname"]
+ person.born = record["born"] if record["born"] and record["born"] != ""
+ person.died = record["died"] if record["died"] and record["died"] != ""
+
+ person.url = record["url"]
+ if person.born and person.born.year > 1900 #restrict the dataset to recent times
+ born = person.born.year.to_s
+ person.id = "#{person.surname}_#{person.forenames[0..0]}_#{born}"
+ person.save
+ end
+ end
+ end
+end
View
12 models/person.rb
@@ -0,0 +1,12 @@
+require 'mongo_mapper'
+
+class Person
+ include MongoMapper::Document
+
+ key :title, String
+ key :forenames, String
+ key :surname, String
+ key :born, Date
+ key :died, Date
+ key :url, String
+end
Please sign in to comment.
Something went wrong with that request. Please try again.