Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allow spotting of 'redirect' entries #869

Merged
merged 2 commits into from
May 21, 2015
Merged

Allow spotting of 'redirect' entries #869

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e5f0148
Fix whitespace.
dracos May 20, 2015
b05b793
Utilise any 'redirect' entries in JSON data.
dracos May 20, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 18 additions & 3 deletions scripts/load-people
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ verbose("End");

# ---

my ($dbh, $memberadd, $memberexist, $membercheck, $nameadd, $nameupdate, $namefetch, $namedelete);
my ($dbh, $memberadd, $memberexist, $membercheck, $nameadd, $nameupdate, $namefetch, $namedelete, $gradd, $grdelete);

sub db_connect {
#DBI->trace(1);
Expand All @@ -73,6 +73,9 @@ sub db_connect {
family_name, lordofname, start_date, end_date) values (?, ?, ?, ?, ?, ?, ?, ?)");
$namefetch = $dbh->prepare("select id from person_names where person_id = ? and end_date >= ? ORDER BY id");
$namedelete = $dbh->prepare('delete from person_names where id=?');

$gradd = $dbh->prepare("replace into gidredirect (gid_from, gid_to) values (?,?)");
$grdelete = $dbh->prepare("delete from gidredirect where gid_from = ?");
}

my %organizations;
Expand Down Expand Up @@ -142,19 +145,29 @@ sub load_people {
$posts{$_->{id}} = $_;
}
foreach (@{$j->{memberships}}) {
if ($_->{redirect}) {
$gradd->execute($_->{id}, $_->{redirect});
} else {
load_member($_);
}
}
foreach (@{$j->{persons}}) {
if ($_->{redirect}) {
$gradd->execute($_->{id}, $_->{redirect});
} else {
load_names($_);
}
}
}

my %member_ids = ();

sub load_member {
my ($member) = @_;

(my $id = $member->{id}) =~ s:uk.org.publicwhip/(member|lord|royal)/::;
my $id = $member->{id};
$grdelete->execute($id);
$id =~ s:uk.org.publicwhip/(member|lord|royal)/::;
(my $person_id = $member->{person_id}) =~ s#uk.org.publicwhip/person/##;

$member_ids{$id} = 1;
Expand Down Expand Up @@ -195,7 +208,9 @@ sub load_member {
sub load_names {
my $person = shift;

(my $id = $person->{id}) =~ s#uk.org.publicwhip/person/##;
my $id = $person->{id};
$grdelete->execute($id);
$id =~ s#uk.org.publicwhip/person/##;

my @names;
foreach my $name (@{$person->{other_names}}) {
Expand Down
Empty file modified scripts/mpinfoin.pl
View file
Open in desktop
Whitespace-only changes.
43 changes: 28 additions & 15 deletions scripts/xml2db.pl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ END
use vars qw($hpos $curdate);
use vars qw($currsection $currsubsection $inoralanswers $promotedheading);
use vars qw(%gids %grdests %ignorehistorygids $tallygidsmode $tallygidsmodedummycount);
use vars qw(%membertoperson);
use vars qw(%membertoperson %personredirect);
use vars qw($current_file);

use vars qw($debatesdir $wransdir $lordswransdir $westminhalldir $wmsdir
Expand Down Expand Up @@ -267,9 +267,20 @@ sub process_type {
my $pwmembers = mySociety::Config::get('PWMEMBERS');
my $j = decode_json(read_file($pwmembers . 'people.json'));
foreach (@{$j->{memberships}}) {
next if $_->{redirect};
(my $person_id = $_->{person_id}) =~ s#uk.org.publicwhip/person/##;
$membertoperson{$_->{id}} = $person_id;
}
foreach (@{$j->{memberships}}) {
next unless $_->{redirect};
$membertoperson{$_->{id}} = $membertoperson{$_->{redirect}};
}
foreach (@{$j->{persons}}) {
next unless $_->{redirect};
(my $id = $_->{id}) =~ s#uk.org.publicwhip/person/##;
(my $redirect = $_->{redirect}) =~ s#uk.org.publicwhip/person/##;
$personredirect{$id} = $redirect;
}

# Process main data
process_type(["debates"], [$debatesdir], \&add_debates_day) if ($debates) ;
Expand Down Expand Up @@ -884,6 +895,14 @@ sub last_id
return $arr[0];
}

sub person_id {
my ($item, $member_id_attr) = @_;
my $person_id = $item->att('person_id') || $membertoperson{$item->att($member_id_attr)} || 'unknown';
$person_id =~ s/.*\///;
$person_id = $personredirect{$person_id} || $person_id;
return $person_id;
}

##########################################################################
# Written Answers

Expand Down Expand Up @@ -1030,13 +1049,12 @@ sub load_debate_division {
die unless $side =~ /^(aye|no|content|not-content)$/;
$text .= "<h2>\u$side</h2> <ul class='division-list'>";
my @names = $list->children($vote_tag); # attr ids vote (teller), text is name
foreach my $person (@names) {
my $person_id = $person->att('person_id') || $membertoperson{$person->att('id')};
$person_id =~ s/.*\///;
my $vote = $person->att('vote');
foreach my $vote (@names) {
my $person_id = person_id($vote, 'id');
my $vote = $vote->att('vote');
die unless $vote eq $side;
my $teller = $person->att('teller');
my $name = $person->sprint(1);
my $teller = $vote->att('teller');
my $name = $vote->sprint(1);
$name =~ s/ *\[Teller\]//; # In Lords
$name =~ s/^(.*), (.*)$/$2 $1/;
$name =~ s/^(rh|Mr|Sir|Ms|Mrs|Dr) //;
Expand Down Expand Up @@ -1407,7 +1425,7 @@ sub add_standing_day {
foreach (@names) {
my $chairman = ($_->parent()->tag() eq 'chairmen');
my $attending = ($_->att('attending') eq 'true');
my $person_id = $_->att('person_id') || $membertoperson{$_->att('memberid')};
my $person_id = person_id($_, 'memberid');
$current_file =~ /_(\d\d-\d)_/;
my $sitting = $1;
if (my ($id, $curr_attending) = $dbh->selectrow_array('select id,attending from pbc_members where person_id=? and bill_id=?
Expand Down Expand Up @@ -1497,8 +1515,7 @@ sub load_standing_division {
my @names = $division->descendants('mpname');
my %out = ( aye => '', no => '' );
foreach (@names) {
my $person_id = $_->att('person_id') || $membertoperson{$_->att('memberid')};
$person_id =~ s/.*\///;
my $person_id = person_id($_, 'memberid');
my $name = $_->att('membername');
my $v = $_->att('vote');
$out{$v} .= '<a href="/mp/?p=' . $person_id . '">' . $name . '</a>, ';
Expand Down Expand Up @@ -1661,11 +1678,7 @@ sub do_load_speech
my $pretext = "";
if ($speech->att('person_id') || $speech->att('speakerid')) {
$type = 12;
if ($speech->att('person_id')) {
($speaker = $speech->att('person_id')) =~ s#uk.org.publicwhip/person/##;
} else {
$speaker = $membertoperson{$speech->att('speakerid')} || 'unknown';
}
$speaker = person_id($speech, 'speakerid');
if ($speaker eq "unknown") {
$speaker = 0;
my $encoded = HTML::Entities::encode_entities($speech->att('speakername'));
Expand Down
14 changes: 14 additions & 0 deletions www/includes/easyparliament/member.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,13 @@ public function MEMBER($args) {
$person_id = $this->postcode_to_person_id($args['postcode'], $house);
} elseif (isset($args['person_id']) && is_numeric($args['person_id'])) {
$person_id = $args['person_id'];
$q = $this->db->query("SELECT gid_to FROM gidredirect
WHERE gid_from = :gid_from",
array(':gid_from' => "uk.org.publicwhip/person/$person_id")
);
if ($q->rows > 0) {
$person_id = str_replace('uk.org.publicwhip/person/', '', $q->field(0, 'gid_to'));
}
}

if (!$person_id) {
Expand Down Expand Up @@ -183,6 +190,13 @@ public function member_id_to_person_id($member_id) {
WHERE member_id = :member_id",
array(':member_id' => $member_id)
);
if ($q->rows == 0) {
$q = $this->db->query("SELECT person_id FROM gidredirect, member
WHERE gid_from = :gid_from AND
CONCAT('uk.org.publicwhip/member/', member_id) = gid_to",
array(':gid_from' => "uk.org.publicwhip/member/$member_id")
);
}
if ($q->rows > 0) {
return $q->field(0, 'person_id');
} else {
Expand Down
15 changes: 15 additions & 0 deletions www/includes/easyparliament/searchengine.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,15 @@ public function SEARCHENGINE($query) {
$this->query .= " 19990101..$to";
}

# Merged people
$db = new ParlDB;
$merged = $db->query('SELECT * FROM gidredirect WHERE gid_from LIKE :gid_from', array(':gid_from' => "uk.org.publicwhip/person/%"));
for ($n=0; $n<$merged->rows(); $n++) {
$from_id = str_replace('uk.org.publicwhip/person/', '', $merged->field($n, 'gid_from'));
$to_id = str_replace('uk.org.publicwhip/person/', '', $merged->field($n, 'gid_to'));
$this->query = preg_replace("#speaker:($from_id|$to_id)#i", "(speaker:$from_id OR speaker:$to_id)", $this->query);
}

twfy_debug("SEARCH", "prefixed: " . var_export($this->prefixed, true));

twfy_debug("SEARCH", "query -- ". $this->query);
Expand Down Expand Up @@ -253,6 +262,12 @@ public function SEARCHENGINE($query) {
}

# Speakers
for ($n=0; $n<$merged->rows(); $n++) {
$from_id = str_replace('uk.org.publicwhip/person/', '', $merged->field($n, 'gid_from'));
$to_id = str_replace('uk.org.publicwhip/person/', '', $merged->field($n, 'gid_to'));
$qd = preg_replace("#\(S$from_id OR S$to_id\)#", "S$to_id", $qd);
}

preg_match_all('#S(\d+)#', $qd, $m);
foreach ($m[1] as $mm) {
$member = new MEMBER(array('person_id' => $mm));
Expand Down