Skip to content

Commit

Permalink
Github updates.
Browse files Browse the repository at this point in the history
* Add some sanity checking to make sure the github feed gave us data from the right project.  (They apparently had a temporary bug, and the consequences were quite spammy.)
* Add a warning message and return semigracefully if a feed no longer exists.
* Move the timer registration down a bit, so github parsers can be tested with modules/local/test.pl.
* The feed format changed ever so slightly (they added a rel attribute to the link tag), add that to the testsuite just in case.
  • Loading branch information
Infinoid committed Feb 28, 2010
1 parent e63d944 commit 27766d4
Show file tree
Hide file tree
Showing 2 changed files with 47 additions and 4 deletions.
16 changes: 14 additions & 2 deletions modules/local/githubparser.pm
Expand Up @@ -108,6 +108,10 @@ sub process_branch {
my $atom = XML::Atom::Client->new(); my $atom = XML::Atom::Client->new();
$feed = $atom->getFeed($$self{branches}{$branch}{url}); $feed = $atom->getFeed($$self{branches}{$branch}{url});
} }
if(!defined($feed)) {
warn "could not fetch branch $branch feed " . $$self{branches}{$branch}{url};
return;
}


my @items = $feed->entries; my @items = $feed->entries;
@items = sort { $a->updated cmp $b->updated } @items; # ascending order @items = sort { $a->updated cmp $b->updated } @items; # ascending order
Expand All @@ -118,12 +122,15 @@ sub process_branch {
foreach my $item (@items) { foreach my $item (@items) {
my $link = $item->link->href; my $link = $item->link->href;
my ($rev) = $link =~ m|/commit/([a-z0-9]{40})|; my ($rev) = $link =~ m|/commit/([a-z0-9]{40})|;
my ($proj) = $link =~ m|http://github.com/[^/]+/([^/]+)/|;
if(exists($$self{not_first_time})) { if(exists($$self{not_first_time})) {
return unless $proj = $$self{project};
# output new entries to channel # output new entries to channel
next if exists($$self{seen}{$rev}); next if exists($$self{seen}{$rev});
$$self{seen}{$rev} = 1; $$self{seen}{$rev} = 1;
$self->output_item($item, $branch, $link, $rev); $self->output_item($item, $branch, $link, $rev);
} else { } else {
die "got bad data from github feed" unless $proj = $$self{project};
# just populate the seen cache # just populate the seen cache
$$self{seen}{$rev} = 1; $$self{seen}{$rev} = 1;
} }
Expand Down Expand Up @@ -206,6 +213,7 @@ sub try_link {


# create project, if necessary # create project, if necessary
my $self = $objects_by_package{$modulename}; my $self = $objects_by_package{$modulename};
my $register_timer = 0;
if(!defined($self)) { if(!defined($self)) {
$objects_by_package{$modulename} = $self = { $objects_by_package{$modulename} = $self = {
project => $project, project => $project,
Expand All @@ -216,8 +224,7 @@ sub try_link {
# create a dynamic subclass to get the timer callback back to us # create a dynamic subclass to get the timer callback back to us
eval "package $modulename; use base 'modules::local::githubparser';"; eval "package $modulename; use base 'modules::local::githubparser';";
$objects_by_package{$modulename} = bless($self, $modulename); $objects_by_package{$modulename} = bless($self, $modulename);
main::create_timer($parsername."_process_project_timer", $modulename, $register_timer = 1;
"process_project", 300 + $feed_number++);
main::lprint("github: created project $project ($modulename)"); main::lprint("github: created project $project ($modulename)");
} }


Expand All @@ -244,6 +251,11 @@ sub try_link {
main::lprint("github: $project/$branchname will output to ".join("/",@$target)); main::lprint("github: $project/$branchname will output to ".join("/",@$target));
} }
} }

if($register_timer) {
main::create_timer($parsername."_process_project_timer", $modulename,
"process_project", 300 + $feed_number++);
}
} }




Expand Down
35 changes: 33 additions & 2 deletions t/rakudolog.t
Expand Up @@ -82,10 +82,10 @@ $output = [output()];
is(scalar @$output, 6, "6 lines of output"); is(scalar @$output, 6, "6 lines of output");
is($$output[0]{net} , 'magnet' , "line to magnet/#parrot"); is($$output[0]{net} , 'magnet' , "line to magnet/#parrot");
is($$output[0]{chan}, '#parrot' , "line to magnet/#parrot"); is($$output[0]{chan}, '#parrot' , "line to magnet/#parrot");
like($$output[0]{text}, qr|rakudo/master: |, "master branch"); like($$output[0]{text}, qr|rakudo: |, "master branch");
is($$output[1]{net} , 'freenode', "line to freenode/#perl6"); is($$output[1]{net} , 'freenode', "line to freenode/#perl6");
is($$output[1]{chan}, '#perl6' , "line to freenode/#perl6"); is($$output[1]{chan}, '#perl6' , "line to freenode/#perl6");
like($$output[1]{text}, qr|rakudo/master: |, "master branch"); like($$output[1]{text}, qr|rakudo: |, "master branch");
BEGIN { $tests += 7 }; BEGIN { $tests += 7 };


# update with multiple commits having the same timestamp # update with multiple commits having the same timestamp
Expand Down Expand Up @@ -131,3 +131,34 @@ my @message_list = ($$output[2]{text}, $$output[8]{text});
is(scalar grep(/Small optimizations/ , @message_list), 1, "log message"); is(scalar grep(/Small optimizations/ , @message_list), 1, "log message");
is(scalar grep(/Add some micro-bench/, @message_list), 1, "log message"); is(scalar grep(/Add some micro-bench/, @message_list), 1, "log message");
BEGIN { $tests += 7 }; BEGIN { $tests += 7 };

# update with their post-2010-02-26 feed format (added rel attribute to the <link> tag)
reset_output();
$xml_footer = << '__XML__' . $xml_footer;
<entry>
<id>tag:github.com,2008:Grit::Commit/b131f6052a181bdad8f7b9e5abe30c9c2c2e360e</id>
<link type="text/html" href="http://github.com/rakudo/rakudo/commit/b131f6052a181bdad8f7b9e5abe30c9c2c2e360e" rel="alternate"/>
<title>Split .defined method and defined vtable method for language interop, as per pmichaud++.</title>
<updated>2010-02-27T16:46:53-08:00</updated>
<content type="html">&lt;pre&gt;m src/builtins/Parcel.pir
Split .defined method and defined vtable method for language interop, as per pmichaud++.&lt;/pre&gt;</content>
<author>
<name>Jonathan Worthington</name>
</author>
</entry>
__XML__
$xml = $xml_header . '<updated>2010-02-27T16:46:53-08:00</updated>' . $xml_footer;
$feed = XML::Atom::Feed->new(\$xml);
call_func('process_branch', 'master', $feed);
$output = [output()];
is(scalar @$output, 6, "6 lines of output");
is($$output[0]{net} , 'magnet' , "line to magnet/#parrot");
is($$output[0]{chan}, '#parrot' , "line to magnet/#parrot");
is($$output[1]{net} , 'freenode', "line to freenode/#perl6");
is($$output[1]{chan}, '#perl6' , "line to freenode/#perl6");
# The module sorts by <updated> time, but the time is the same for these two commits.
# Do it this way so we don't depend on perl's internal sort algorithm details.
@message_list = ($$output[2]{text});
is(scalar grep(/Split .defined method/ , @message_list), 1, "log message");
BEGIN { $tests += 6 };

0 comments on commit 27766d4

Please sign in to comment.