Skip to content

Commit

Permalink
Utilise any 'redirect' entries in JSON data.
Browse files Browse the repository at this point in the history 
This allows us to load in people that have been merged without having to
update every relevant XML file or search index, and redirect at the
front end.
  • Loading branch information
@dracos
dracos committed May 21, 2015
1 parent e5f0148 commit b05b793
Show file tree
Hide file tree
Showing 4 changed files with 80 additions and 23 deletions.
25 changes: 20 additions & 5 deletions scripts/load-people
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ verbose("End");

# ---

my ($dbh, $memberadd, $memberexist, $membercheck, $nameadd, $nameupdate, $namefetch, $namedelete);
my ($dbh, $memberadd, $memberexist, $membercheck, $nameadd, $nameupdate, $namefetch, $namedelete, $gradd, $grdelete);

sub db_connect {
#DBI->trace(1);
Expand All @@ -73,6 +73,9 @@ sub db_connect {
family_name, lordofname, start_date, end_date) values (?, ?, ?, ?, ?, ?, ?, ?)");
$namefetch = $dbh->prepare("select id from person_names where person_id = ? and end_date >= ? ORDER BY id");
$namedelete = $dbh->prepare('delete from person_names where id=?');

$gradd = $dbh->prepare("replace into gidredirect (gid_from, gid_to) values (?,?)");
$grdelete = $dbh->prepare("delete from gidredirect where gid_from = ?");
}

my %organizations;
Expand Down Expand Up @@ -142,10 +145,18 @@ sub load_people {
$posts{$_->{id}} = $_;
}
foreach (@{$j->{memberships}}) {
load_member($_);
if ($_->{redirect}) {
$gradd->execute($_->{id}, $_->{redirect});
} else {
load_member($_);
}
}
foreach (@{$j->{persons}}) {
load_names($_);
if ($_->{redirect}) {
$gradd->execute($_->{id}, $_->{redirect});
} else {
load_names($_);
}
}
}

Expand All @@ -154,7 +165,9 @@ my %member_ids = ();
sub load_member {
my ($member) = @_;

(my $id = $member->{id}) =~ s:uk.org.publicwhip/(member|lord|royal)/::;
my $id = $member->{id};
$grdelete->execute($id);
$id =~ s:uk.org.publicwhip/(member|lord|royal)/::;
(my $person_id = $member->{person_id}) =~ s#uk.org.publicwhip/person/##;

$member_ids{$id} = 1;
Expand Down Expand Up @@ -195,7 +208,9 @@ sub load_member {
sub load_names {
my $person = shift;

(my $id = $person->{id}) =~ s#uk.org.publicwhip/person/##;
my $id = $person->{id};
$grdelete->execute($id);
$id =~ s#uk.org.publicwhip/person/##;

my @names;
foreach my $name (@{$person->{other_names}}) {
Expand Down
43 changes: 28 additions & 15 deletions scripts/xml2db.pl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ END
use vars qw($hpos $curdate);
use vars qw($currsection $currsubsection $inoralanswers $promotedheading);
use vars qw(%gids %grdests %ignorehistorygids $tallygidsmode $tallygidsmodedummycount);
use vars qw(%membertoperson);
use vars qw(%membertoperson %personredirect);
use vars qw($current_file);

use vars qw($debatesdir $wransdir $lordswransdir $westminhalldir $wmsdir
Expand Down Expand Up @@ -267,9 +267,20 @@ sub process_type {
my $pwmembers = mySociety::Config::get('PWMEMBERS');
my $j = decode_json(read_file($pwmembers . 'people.json'));
foreach (@{$j->{memberships}}) {
next if $_->{redirect};
(my $person_id = $_->{person_id}) =~ s#uk.org.publicwhip/person/##;
$membertoperson{$_->{id}} = $person_id;
}
foreach (@{$j->{memberships}}) {
next unless $_->{redirect};
$membertoperson{$_->{id}} = $membertoperson{$_->{redirect}};
}
foreach (@{$j->{persons}}) {
next unless $_->{redirect};
(my $id = $_->{id}) =~ s#uk.org.publicwhip/person/##;
(my $redirect = $_->{redirect}) =~ s#uk.org.publicwhip/person/##;
$personredirect{$id} = $redirect;
}

# Process main data
process_type(["debates"], [$debatesdir], \&add_debates_day) if ($debates) ;
Expand Down Expand Up @@ -884,6 +895,14 @@ sub last_id
return $arr[0];
}

sub person_id {
my ($item, $member_id_attr) = @_;
my $person_id = $item->att('person_id') || $membertoperson{$item->att($member_id_attr)} || 'unknown';
$person_id =~ s/.*\///;
$person_id = $personredirect{$person_id} || $person_id;
return $person_id;
}

##########################################################################
# Written Answers

Expand Down Expand Up @@ -1030,13 +1049,12 @@ sub load_debate_division {
die unless $side =~ /^(aye|no|content|not-content)$/;
$text .= "<h2>\u$side</h2> <ul class='division-list'>";
my @names = $list->children($vote_tag); # attr ids vote (teller), text is name
foreach my $person (@names) {
my $person_id = $person->att('person_id') || $membertoperson{$person->att('id')};
$person_id =~ s/.*\///;
my $vote = $person->att('vote');
foreach my $vote (@names) {
my $person_id = person_id($vote, 'id');
my $vote = $vote->att('vote');
die unless $vote eq $side;
my $teller = $person->att('teller');
my $name = $person->sprint(1);
my $teller = $vote->att('teller');
my $name = $vote->sprint(1);
$name =~ s/ *\[Teller\]//; # In Lords
$name =~ s/^(.*), (.*)$/$2 $1/;
$name =~ s/^(rh|Mr|Sir|Ms|Mrs|Dr) //;
Expand Down Expand Up @@ -1407,7 +1425,7 @@ sub add_standing_day {
foreach (@names) {
my $chairman = ($_->parent()->tag() eq 'chairmen');
my $attending = ($_->att('attending') eq 'true');
my $person_id = $_->att('person_id') || $membertoperson{$_->att('memberid')};
my $person_id = person_id($_, 'memberid');
$current_file =~ /_(\d\d-\d)_/;
my $sitting = $1;
if (my ($id, $curr_attending) = $dbh->selectrow_array('select id,attending from pbc_members where person_id=? and bill_id=?
Expand Down Expand Up @@ -1497,8 +1515,7 @@ sub load_standing_division {
my @names = $division->descendants('mpname');
my %out = ( aye => '', no => '' );
foreach (@names) {
my $person_id = $_->att('person_id') || $membertoperson{$_->att('memberid')};
$person_id =~ s/.*\///;
my $person_id = person_id($_, 'memberid');
my $name = $_->att('membername');
my $v = $_->att('vote');
$out{$v} .= '<a href="/mp/?p=' . $person_id . '">' . $name . '</a>, ';
Expand Down Expand Up @@ -1661,11 +1678,7 @@ sub do_load_speech
my $pretext = "";
if ($speech->att('person_id') || $speech->att('speakerid')) {
$type = 12;
if ($speech->att('person_id')) {
($speaker = $speech->att('person_id')) =~ s#uk.org.publicwhip/person/##;
} else {
$speaker = $membertoperson{$speech->att('speakerid')} || 'unknown';
}
$speaker = person_id($speech, 'speakerid');
if ($speaker eq "unknown") {
$speaker = 0;
my $encoded = HTML::Entities::encode_entities($speech->att('speakername'));
Expand Down
14 changes: 14 additions & 0 deletions www/includes/easyparliament/member.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,13 @@ public function MEMBER($args) {
$person_id = $this->postcode_to_person_id($args['postcode'], $house);
} elseif (isset($args['person_id']) && is_numeric($args['person_id'])) {
$person_id = $args['person_id'];
$q = $this->db->query("SELECT gid_to FROM gidredirect
WHERE gid_from = :gid_from",
array(':gid_from' => "uk.org.publicwhip/person/$person_id")
);
if ($q->rows > 0) {
$person_id = str_replace('uk.org.publicwhip/person/', '', $q->field(0, 'gid_to'));
}
}

if (!$person_id) {
Expand Down Expand Up @@ -183,6 +190,13 @@ public function member_id_to_person_id($member_id) {
WHERE member_id = :member_id",
array(':member_id' => $member_id)
);
if ($q->rows == 0) {
$q = $this->db->query("SELECT person_id FROM gidredirect, member
WHERE gid_from = :gid_from AND
CONCAT('uk.org.publicwhip/member/', member_id) = gid_to",
array(':gid_from' => "uk.org.publicwhip/member/$member_id")
);
}
if ($q->rows > 0) {
return $q->field(0, 'person_id');
} else {
Expand Down
21 changes: 18 additions & 3 deletions www/includes/easyparliament/searchengine.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,15 @@ public function SEARCHENGINE($query) {
$this->query .= " 19990101..$to";
}

# Merged people
$db = new ParlDB;
$merged = $db->query('SELECT * FROM gidredirect WHERE gid_from LIKE :gid_from', array(':gid_from' => "uk.org.publicwhip/person/%"));
for ($n=0; $n<$merged->rows(); $n++) {
$from_id = str_replace('uk.org.publicwhip/person/', '', $merged->field($n, 'gid_from'));
$to_id = str_replace('uk.org.publicwhip/person/', '', $merged->field($n, 'gid_to'));
$this->query = preg_replace("#speaker:($from_id|$to_id)#i", "(speaker:$from_id OR speaker:$to_id)", $this->query);
}

twfy_debug("SEARCH", "prefixed: " . var_export($this->prefixed, true));

twfy_debug("SEARCH", "query -- ". $this->query);
Expand Down Expand Up @@ -253,11 +262,17 @@ public function SEARCHENGINE($query) {
}

# Speakers
for ($n=0; $n<$merged->rows(); $n++) {
$from_id = str_replace('uk.org.publicwhip/person/', '', $merged->field($n, 'gid_from'));
$to_id = str_replace('uk.org.publicwhip/person/', '', $merged->field($n, 'gid_to'));
$qd = preg_replace("#\(S$from_id OR S$to_id\)#", "S$to_id", $qd);
}

preg_match_all('#S(\d+)#', $qd, $m);
foreach ($m[1] as $mm) {
$member = new MEMBER(array('person_id' => $mm));
$name = iconv('iso-8859-1', 'utf-8//TRANSLIT', $member->full_name()); # Names are currently in ISO-8859-1
$qd = str_replace("S$mm", "speaker:$name", $qd);
$member = new MEMBER(array('person_id' => $mm));
$name = iconv('iso-8859-1', 'utf-8//TRANSLIT', $member->full_name()); # Names are currently in ISO-8859-1
$qd = str_replace("S$mm", "speaker:$name", $qd);
}

# Simplify display of excluded words
Expand Down

0 comments on commit b05b793

Please sign in to comment.