Skip to content
Browse files

Input::Twitter: Change timeline traversing strategy due to twitter.co…

…m's change of API policy.

  twitter.com will make "page" query parameter deprecated because it often fails to work properly especially if the timeline is rapid to grow.
  Input::Twitter now uses "max_id" parameter to traverse timelines, as recommended by the authority.
  See https://dev.twitter.com/docs/working-with-timelines
  • Loading branch information...
1 parent c52a320 commit 897801c6a01b6646a214b5b2ca207ba0f36d16ea @debug-ito committed
Showing with 42 additions and 66 deletions.
  1. +26 −49 lib/BusyBird/Input/Twitter.pm
  2. +6 −4 lib/BusyBird/Input/Twitter/HomeTimeline.pm
  3. +10 −13 lib/BusyBird/Input/Twitter/List.pm
View
75 lib/BusyBird/Input/Twitter.pm
@@ -66,14 +66,9 @@ sub _setParams {
my ($self, $params_ref) = @_;
$self->SUPER::_setParams($params_ref);
$self->_setParam($params_ref, 'worker', undef, 1);
+ $self->{max_id_for_page} = [];
}
-## sub _getTimeline {
-## my ($self, $count, $page) = @_;
-## ## MUST BE IMPLEMENTED IN SUBCLASSES
-## return undef;
-## }
-
sub _getWorkerInput {
my ($self, $count, $page) = @_;
## MUST BE IMPLEMENTED IN SUBCLASSES
@@ -94,21 +89,39 @@ sub _extractStatusesFromWorkerData {
'screen_name' => $nt_status->{user}->{screen_name},
'name' => $nt_status->{user}->{name},
'profile_image_url' => $nt_status->{user}->{profile_image_url},
- }
+ },
+ busybird => {
+ original_id => $nt_status->{id},
+ original_id_str => $nt_status->{id_str},
+ },
);
push(@statuses, $status);
}
- ## my @statuses = map { BusyBird::Status::Twitter->new($_) } @$worker_data;
return \@statuses;
}
+sub _logWorkerInput {
+ my ($self, $worker_input) = @_;
+ my $argref = $worker_input->{args}->[0];
+ &bblog(sprintf(
+ "%s: method: %s, args: %s", __PACKAGE__, $worker_input->{method},
+ join(", ", map {"$_: " . $argref->{$_}} keys %$argref)
+ ));
+}
+
sub _getStatusesPage {
my ($self, $count, $page, $callback) = @_;
+ if($page <= 0) {
+ @{$self->{max_id_for_page}} = ();
+ }elsif(!defined($self->{max_id_for_page}[$page])) {
+ &bblog(sprintf("%s: max_id for page $page is undefined. Something's wrong."), __PACKAGE__);
+ }
my $worker_input = $self->_getWorkerInput($count, $page);
if(!$worker_input) {
$callback->(undef);
return;
}
+ $self->_logWorkerInput($worker_input);
$worker_input->{cb} = sub {
my ($status, @data) = @_;
if($status != BusyBird::Worker::Object::STATUS_OK) {
@@ -116,50 +129,14 @@ sub _getStatusesPage {
$callback->(undef);
return;
}
- $callback->($self->_extractStatusesFromWorkerData($data[0]));
+ my $bb_status = $self->_extractStatusesFromWorkerData($data[0]);
+ if(@$bb_status) {
+ $self->{max_id_for_page}->[$page + 1] = $bb_status->[$#$bb_status]->content->{busybird}{original_id_str};
+ }
+ $callback->($bb_status);
};
$self->{worker}->startJob(%$worker_input);
-
- ## my ($self, $callstack, $ret_session, $ret_event, $count, $page) = @_;
- ## my $worker_input = $self->_getWorkerInput($count, $page);
- ## if(!$worker_input) {
- ## POE::Kernel->post($ret_session, $ret_event, $callstack, undef);
- ## return;
- ## }
- ## BusyBird::CallStack->newStack($callstack, $ret_session, $ret_event, count => $count, page => $page);
- ## $self->{worker}->startJob($callstack, $self->{session}, 'on_worker_complete', $worker_input);
- ## return;
-
- ## my ($self, $count, $page) = @_;
- ## my $timeline = $self->_getTimeline($count, $page);
- ## printf STDERR ("DEBUG: Got %d tweets from input %s\n", int(@$timeline), $self->getName());
- ## return undef if !$timeline;
- ## ## foreach my $status (@$timeline) {
- ## ## push(@$ret_stats, BusyBird::Status::Twitter->new($status));
- ## ## }
- ## my @ret_stats = map { BusyBird::Status::Twitter->new($_) } @$timeline;
- ## return \@ret_stats;
}
-## sub _sessionStart {
-## my ($self, $kernel) = @_[OBJECT, KERNEL];
-## $self->SUPER::_sessionStart(@_[1 .. $#_]);
-## $kernel->state('on_worker_complete', $self, '_sessionOnWorkerComplete');
-## }
-##
-## sub _sessionOnWorkerComplete {
-## my ($self, $kernel, $callstack, $output_objs, $input_obj, $exit_status) = @_[OBJECT, KERNEL, ARG0 .. ARG3];
-## ## print STDERR ("BusyBird::Input::Twitter: Worker complate!------\n");
-## ## print STDERR (Dumper($output_objs));
-## ## print STDERR ("-----------------\n");
-## my ($worker_status, $worker_data) = ($output_objs->[0]->{status}, $output_objs->[0]->{data});
-## if($worker_status != BusyBird::Worker::Object::STATUS_OK) {
-## &bblog(sprintf("WARNING: Twitter worker returns worker_status %d", $worker_status));
-## $callstack->pop(undef);
-## return;
-## }
-## &bblog(sprintf("DEBUG: Got %d tweets from input %s", int(@$worker_data), $self->getName()));
-## $callstack->pop($self->_extractStatusesFromWorkerData($worker_data));
-## }
1;
View
10 lib/BusyBird/Input/Twitter/HomeTimeline.pm
@@ -6,12 +6,14 @@ use base ('BusyBird::Input::Twitter');
sub _getWorkerInput {
my ($self, $count, $page) = @_;
- ## return $self->{nt}->home_timeline({count => $count, page => $page});
- return {method => 'home_timeline', context => 'scalar', args => [{
+ my $args = {
count => $count,
- page => $page + 1,
include_entities => 1,
- }]};
+ };
+ if(defined($self->{max_id_for_page}[$page])) {
+ $args->{max_id} = $self->{max_id_for_page}[$page];
+ }
+ return {method => 'home_timeline', context => 'scalar', args => [$args]};
}
1;
View
23 lib/BusyBird/Input/Twitter/List.pm
@@ -13,20 +13,17 @@ sub _setParams {
sub _getWorkerInput {
my ($self, $count, $page) = @_;
- ## return $self->{nt}->list_statuses({
- ## user => $self->{owner_name},
- ## list_id => $self->{list_slug_name},
- ## per_page => $count,
- ## page => $page,
- ## });
+ my $args = {
+ user => $self->{owner_name},
+ list_id => $self->{list_slug_name},
+ per_page => $count,
+ include_entities => 1,
+ };
+ if(defined($self->{max_id_for_page}[$page])) {
+ $args->{max_id} = $self->{max_id_for_page}[$page];
+ }
return {method => 'list_statuses', context => 's',
- args => [{
- user => $self->{owner_name},
- list_id => $self->{list_slug_name},
- per_page => $count,
- page => $page + 1,
- include_entities => 1,
- }]};
+ args => [$args]};
}
1;

0 comments on commit 897801c

Please sign in to comment.
Something went wrong with that request. Please try again.