Skip to content

Commit

Permalink
reorganize site assets per domain.
Browse files Browse the repository at this point in the history
  Filter-FindEnclosures/youtube.pl -> youtube.com/find_enclosures.pl
  CustomFeed-FindLinks/asahicom.yaml -> asahi.com/find_links.yaml
  • Loading branch information
miyagawa committed Jan 10, 2009
1 parent 78c4aed commit dfd42de
Show file tree
Hide file tree
Showing 57 changed files with 38 additions and 91 deletions.
50 changes: 27 additions & 23 deletions lib/Plagger/Plugin.pm
Expand Up @@ -46,21 +46,23 @@ sub plugin_id {

sub assets_dir {
my $self = shift;
my $context = Plagger->context;

if ($self->conf->{assets_path}) {
return $self->conf->{assets_path}; # look at config:assets_path first
}
my $context = Plagger->context;

my $assets_base =
$context->conf->{assets_path} || # or global:assets_path
File::Spec->catfile($FindBin::Bin, "assets"); # or "assets" under plagger script

return File::Spec->catfile(
$assets_base, "plugins", $self->class_id,
$assets_base, "plugins",
);
}

sub asset_key {
my $self = shift;
return $self->plugin_id;
}

sub log {
my $self = shift;
Plagger->context->log(@_, caller => ref($self));
Expand All @@ -78,11 +80,10 @@ sub cookie_jar {
}

sub load_assets {
my($self, $rule, $callback) = @_;
my($self, $ext, $callback) = @_;

unless (blessed($rule) && $rule->isa('File::Find::Rule')) {
$rule = File::Find::Rule->name($rule)->extras({follow => 1});
}
my $key = $self->asset_key;
my $rule = File::Find::Rule->name("$key*.$ext")->extras({ follow => 1 });

# ignore .svn directories
$rule->or(
Expand All @@ -91,28 +92,31 @@ sub load_assets {
);

# $rule isa File::Find::Rule
for my $file ($rule->in($self->assets_dir)) {
my $base = File::Basename::basename($file);
$callback->($file, $base);
my $assets_dir = $self->assets_dir;
for my $file ($rule->in($assets_dir)) {
my $domain = File::Basename::dirname($file);
$domain =~ s/^$assets_dir\///;
$domain = '*' if $domain eq 'default';
$callback->($file, $domain);
}
}

sub add_plugin {
sub add_asset {
my $self = shift;
my($plugin) = @_;

my $domain = $plugin->{domain} || '*';
push @{ $self->{plugins}->{$domain} }, $plugin;
push @{ $self->{assets}->{$domain} }, $plugin;
}

sub plugin_for {
sub asset_for {
my $self = shift;
my($url) = @_;

return $self->plugins_for($url, 1);
return $self->assets_for($url, 1);
}

sub plugins_for {
sub assets_for {
my $self = shift;
my($url, $first) = @_;

Expand All @@ -125,22 +129,22 @@ sub plugins_for {
my @try = map join(".", @domain[$_..$#domain]), 0..$#domain-1;
push @try, '*';

my @plugins;
my @assets;
for my $try (@try) {
my $plugins = $self->{plugins}->{$try} || [];
for my $plugin (@{$plugins}) {
my $assets = $self->{assets}->{$try} || [];
for my $plugin (@{$assets}) {
my $re = $plugin->{handle} || ".";
my $test = $re =~ m!https?://! ? $uri : $uri->path_query;
if ($test =~ /$re/i) {
$self->log(debug => "Handle $uri with plugin " . $plugin->site_name);
$self->log(debug => "Handle $uri with plugin " . $plugin->{domain});
return $plugin if $first;
push @plugins, $plugin;
push @assets, $plugin;
}
}
}

return if $first;
return @plugins;
return @assets;
}

1;
Expand Down
21 changes: 9 additions & 12 deletions lib/Plagger/Plugin/CustomFeed/FindLinks.pm
Expand Up @@ -22,27 +22,29 @@ sub register {
sub init {
my $self = shift;
$self->SUPER::init(@_);
$self->load_assets('*.yaml', sub { $self->load_plugin_yaml(@_) });
$self->load_assets('yaml', sub { $self->load_plugin_yaml(@_) });

$self->{ua} = Plagger::UserAgent->new;
}

sub asset_key { 'find_links' }

sub load_plugin_yaml {
my($self, $file, $base) = @_;
my($self, $file, $domain) = @_;

Plagger->context->log(debug => "Load YAML $file");
my @data = YAML::LoadFile($file);

for my $data (@data) {
my $plugin = Plagger::Plugin::Filter::FindLinks::YAML->new($data, $base);
$self->add_plugin($plugin);
my $plugin = Plagger::Plugin::Filter::FindLinks::YAML->new($data, $domain);
$self->add_asset($plugin);
}
}

sub handle {
my($self, $context, $args) = @_;

my $handler = $self->plugin_for($args->{feed}->url);
my $handler = $self->asset_for($args->{feed}->url);
if ($handler) {
$args->{match} = $handler->follow_link;
$args->{xpath} = $handler->follow_xpath;
Expand Down Expand Up @@ -140,20 +142,15 @@ use Encode;
use List::Util qw(first);

sub new {
my($class, $data, $base) = @_;
my($class, $data, $domain) = @_;

# add ^ if handle method starts with http://
for my $key ( qw(handle handle_force) ) {
next unless defined $data->{$key};
$data->{$key} = "^$data->{$key}" if $data->{$key} =~ m!^https?://!;
}

bless {%$data, base => $base }, $class;
}

sub site_name {
my $self = shift;
$self->{base};
bless {%$data, domain => $domain }, $class;
}

sub follow_link {
Expand Down
@@ -1,5 +1,4 @@
author: Tatsuhiko Miyagawa
domain: 74.53.95.28
follow_xpath: //a[contains(text(), "720P")]
# TODO item title becomes "HD 720P" and not movie title :/

Expand Down
3 changes: 0 additions & 3 deletions root/plugins/CustomFeed-FindLinks/impresswatch.yaml

This file was deleted.

@@ -1,6 +1,5 @@
sub init {
my $self = shift;
$self->{domain} = "anime.livedoor.com";
$self->{handle} = "movie";
}

Expand Down
Expand Up @@ -3,7 +3,6 @@
use Web::Scraper;
sub init {
my $self = shift;
$self->{domain} = "www.apple.com";
$self->{handle} = "/trailers/";
}

Expand Down
@@ -1,7 +1,6 @@
# author: Tatsuhiko Miyagawa
sub init {
my $self = shift;
$self->{domain} = "www.asahi.com";
$self->{handle} = 'video/\w+/\w{3}\d+\.html';
}

Expand Down
@@ -1,5 +1,4 @@
# upgrades http://www.asahi.com/video/list.html
# author: otsune
domain: www.asahi.com
handle: /video
follow_link: \w{3}\d{12}\.html
Expand Up @@ -2,7 +2,6 @@
# author: Tatsuhiko Miyagawa
sub init {
my $self = shift;
$self->{domain} = "cancam.tv";
$self->{handle} = "/content/models_room";
}

Expand Down
Expand Up @@ -2,7 +2,6 @@
# e.g. http://feeds.cbsnews.com/CBSNewsVideoEveningNews
sub init {
my $self = shift;
$self->{domain} = "cbsnews.com";
$self->{handle} = "/video/watch/";
}

Expand Down
Expand Up @@ -3,7 +3,6 @@

sub init {
my $self = shift;
$self->{domain} = "channel.pandora.tv";
$self->{handle} = "channel/video.ptv";
}

Expand Down
@@ -1,7 +1,6 @@
# author: Tatsuhiko Miyagawa
sub init {
my $self = shift;
$self->{domain} = "colbertnation.com";
$self->{handle} = 'videos/\d+';
}

Expand Down
@@ -1,7 +1,6 @@
# author: Tatsuhiko Miyagawa
sub init {
my $self = shift;
$self->{domain} = "comedycentral.com";
$self->{handle} = 'videos/index\.jhtml\?videoId=';
}

Expand Down
Expand Up @@ -2,7 +2,6 @@

sub init {
my $self = shift;
$self->{domain} = "current.com";
$self->{handle} = '/items/\d+/';
}

Expand Down
@@ -1,5 +1,4 @@
# upgrades http://www.asahi.com/video/list.html
# author: otsune
domain: current.com
handle: .
follow_link: /items/\d+/
Expand Up @@ -2,7 +2,6 @@

sub init {
my $self = shift;
$self->{domain} = "dailymotion.com";
$self->{handle} = "/video/";
}

Expand Down
Expand Up @@ -2,7 +2,6 @@
use XML::LibXML;
sub init {
my $self = shift;
$self->{domain} = "*";
$self->{handle} = "\.asx"; # XXX this really needs to be the handle for upgrade, not find
}

Expand Down
Expand Up @@ -2,7 +2,6 @@
# TODO replace the original video/3gpp
sub init {
my $self = shift;
$self->{domain} = "eyevio.jp";
$self->{handle} = "/movie/";
}

Expand Down
@@ -1,7 +1,6 @@
# author: Tatsuhiko Miyagawa
sub init {
my $self = shift;
$self->{domain} = "fnn-news.com";
$self->{handle} = "/news/headlines/articles";
}

Expand Down
@@ -1,5 +1,4 @@
# upgrades http://www.fnn-news.com/news/headlines/category05.html
# check "Skip RSS discovery"
domain: fnn-news.com
handle: /news/headlines/
follow_link: /news/headlines/articles/
@@ -1,7 +1,6 @@
# author: Tatsuhiko Miyagawa
sub init {
my $self = shift;
$self->{domain} = "funnyordie.com";
$self->{handle} = 'videos/\w+/';
}

Expand Down
@@ -1,7 +1,6 @@
# upgrades http://www.gametrailers.com/rssgenerate.php?game1id=6364&orderby=newest&limit=20
sub init {
my $self = shift;
$self->{domain} = "gametrailers.com";
$self->{handle} = '/player/\d+.html';
}

Expand Down
@@ -1,6 +1,5 @@
sub init {
my $self = shift;
$self->{domain} = "joost.com";
$self->{handle} = ".";
}

Expand Down
@@ -1,5 +1,4 @@
author: Tatsuhiko Miyagawa
domain: lantis-net.com
follow_link: h.asx


Expand Down
@@ -1,5 +1,4 @@
author: Tatsuhiko Miyagawa
domain: macrossf.com
handle: /radio/radio.html
follow_link: 500k.asx

Expand Down
@@ -1,7 +1,6 @@
use Web::Scraper;
sub init {
my $self = shift;
$self->{domain} = "mitter.jp";
$self->{handle} = '/watched/\d+';
}

Expand Down
@@ -1,7 +1,6 @@
# author: Tatsuhiko Miyagawa
sub init {
my $self = shift;
$self->{domain} = "mtvmusic.com";
$self->{handle} = 'videos/\d+';
}

Expand Down
@@ -1,3 +1,2 @@
domain: mtvmusic.com
follow_link: videos/\d+/

@@ -1,7 +1,6 @@
# author: Tatsuhiko Miyagawa
sub init {
my $self = shift;
$self->{domain} = "news.tbs.co.jp";
$self->{handle} = "/newseye/";
}

Expand Down
@@ -1,5 +1,4 @@
# upgrades http://news.tbs.co.jp/society/
# check "Skip RSS discovery"
domain: news.tbs.co.jp
handle: .
follow_xpath: //a[contains(@href,"/newseye/tbs_newseye") and not(contains(text(), "続きを読む"))]
@@ -1,7 +1,6 @@
# author: Tatsuhiko Miyagawa
sub init {
my $self = shift;
$self->{domain} = "www.nhk-ep.co.jp";
$self->{handle} = "/netstar/.*mov_";
}

Expand Down
@@ -1,5 +1,4 @@
author: Tatsuhiko Miyagawa
domain: www.nhk-ep.co.jp
handle: /netstar/
follow_link: mov_
# TODO extract thumbnail from http://www.nhk-ep.co.jp/netstar/kanetomo_contents.html
@@ -1,7 +1,6 @@
# author: Tatsuhiko Miyagawa
sub init {
my $self = shift;
$self->{domain} = "nicovideo.jp";
$self->{handle} = "/watch/";
}

Expand Down
@@ -1,4 +1,3 @@
author: Tatsuhiko Miyagawa
domain: nicovideo.jp
follow_link: /watch/(\d+|\w{2}\d+)

2 comments on commit dfd42de

@miyagawa
Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a backward incompatible change if you have your own custom asset files in root/plugins.

Use http://gist.github.com/45580 and run it

bulkmove.pl find_links root/plugins/CustomFeed-FindLinks/*.yaml
bulkmove.pl find_enclosures root/plugins/Filter-FindEnclosures/*.pl

to move your file automatically. You might need to update “git-mv” line to just “mv”. And then, you don’t need to set ‘domain’ in asset subclasses, so run these to remove those lines:

perl -nle '/domain:/ or print' root/plugins/*/*.yaml                          
perl -i -nle '/{domain}/ or print' root/plugins/*/*.pl

@miyagawa
Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oops, missed -i option.

perl -i -nle '/domain:/ or print' root/plugins/*/*.yaml                          
perl -i -nle '/{domain}/ or print' root/plugins/*/*.pl

Please sign in to comment.