Skip to content

Commit

Permalink
use META.json as well as META.yml
Browse files Browse the repository at this point in the history
  • Loading branch information
@DrHyde
DrHyde committed May 15, 2012
1 parent 057af2b commit 11b5dd4
Show file tree
Hide file tree
Showing 4 changed files with 98 additions and 54 deletions.
33 changes: 22 additions & 11 deletions build-reverse-index.pl
View file
Original file line number Original file line Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
use Data::Dumper; use Data::Dumper;
use FindBin; use FindBin;
use YAML (); use YAML ();
use JSON ();


$/ = undef; $/ = undef;


Expand All @@ -26,20 +27,30 @@
mkdir 'db/reverse'; mkdir 'db/reverse';
chmod 0777, qw(db db/reverse); chmod 0777, qw(db db/reverse);


my %METAyml; my %META;
opendir(DIR, 'db/META.yml') || die("can't open db/META.yml\n"); opendir(DIR, 'db/META.yml') || die("can't open db/META.yml\n");
foreach(grep { -f "db/META.yml/$_" } readdir(DIR)) { foreach(grep { -f "db/META.yml/$_" } readdir(DIR)) {
open FILE, "db/META.yml/$_" || die("Can't read db/META.yml/$_\n"); open FILE, "db/META.yml/$_" || die("Can't read db/META.yml/$_\n");
eval { eval {
$METAyml{$_} = YAML::Load(<FILE>); if($_ =~ /yml$/) {
$METAyml{$_} = join("\n", $META{$_} = YAML::Load(<FILE>);
keys %{$METAyml{$_}->{requires}}, $META{$_} = join("\n",
keys %{$METAyml{$_}->{build_requires}}, keys %{$META{$_}->{requires}},
keys %{$METAyml{$_}->{configure_requires}}, keys %{$META{$_}->{build_requires}},
keys %{$METAyml{$_}->{test_requires}}, keys %{$META{$_}->{configure_requires}},
); keys %{$META{$_}->{test_requires}},
);
} else {
$META{$_} = JSON::decode_json(<FILE>);
$META{$_} = join("\n",
keys %{$META{$_}->{prereqs}->{runtime}->{requires}},
keys %{$META{$_}->{prereqs}->{configure}->{requires}},
keys %{$META{$_}->{prereqs}->{build}->{requires}},
keys %{$META{$_}->{prereqs}->{test}->{requires}},
);
}
}; };
delete $METAyml{$_} if($@); delete $META{$_} if($@);
close(FILE); close(FILE);
} }
closedir(DIR); closedir(DIR);
Expand All @@ -55,8 +66,8 @@
print FILE Dumper([ print FILE Dumper([
sort keys %{{ sort keys %{{
map { s/\.yml$//; s/-v?\d[\d.]*$//; $_ => 1 } map { s/\.yml$//; s/-v?\d[\d.]*$//; $_ => 1 }
grep { $METAyml{$_} =~ /(^|\s)($modules)(\s|$)/ } grep { $META{$_} =~ /(^|\s)($modules)(\s|$)/ }
keys %METAyml keys %META
}} }}
]); ]);
close(FILE); close(FILE);
Expand Down
81 changes: 54 additions & 27 deletions lib/CPANdeps.pm
View file
Original file line number Original file line Diff line number Diff line change
@@ -1,5 +1,3 @@
# $Id: CPANdeps.pm,v 1.37 2009/02/14 23:03:53 drhyde Exp $

package CPANdeps; package CPANdeps;


use strict; use strict;
Expand All @@ -10,6 +8,7 @@ use Cwd;
use CGI; use CGI;
use CGI::Carp qw(fatalsToBrowser); use CGI::Carp qw(fatalsToBrowser);
use YAML (); use YAML ();
use JSON ();
use DBI; use DBI;
use LWP::UserAgent; use LWP::UserAgent;


Expand Down Expand Up @@ -461,36 +460,64 @@ sub getpurity {


sub getreqs { sub getreqs {
my($author, $distname, $ua) = @_; my($author, $distname, $ua) = @_;
my $cachefile = "$home/db/META.yml/$distname.yml"; my $METAymlfile = "$home/db/META.yml/$distname.yml";
my $METAymlURL = "http://search.cpan.org/src/$author/$distname/META.yml"; my $METAjsonfile = "$home/db/META.yml/$distname.json";
my $yaml; my $METAymlURL = "http://search.cpan.org/src/$author/$distname/META.yml";
my $METAjsonURL = "http://search.cpan.org/src/$author/$distname/META.json";
my $meta;
local $/ = undef; local $/ = undef;


# if we have a cache file, read it my $parsed_meta;
if(-e $cachefile) { if(-e $METAymlfile) {
open(YAML, $cachefile) || die("Can't read $cachefile\n"); warn("Cached YAML\n");
$yaml = <YAML>; open(YAML, $METAymlfile) || die("Can't read $METAymlfile\n");
$meta = <YAML>;
close(YAML); close(YAML);
$parsed_meta = eval { YAML::Load($meta); };
} elsif(-e $METAjsonfile) {
warn("Cached JSON\n");
open(JSON, $METAjsonfile) || die("Can't read $METAjsonfile\n");
$meta = <JSON>;
close(JSON);
$parsed_meta = eval { JSON::decode_json($meta); };
} elsif((my $res = $ua->request(HTTP::Request->new(GET => $METAymlURL)))->is_success()) {
warn("Fetching YAML\n");
$meta = $res->content();
open(META, ">$METAymlfile") || die("Can't write $METAymlfile\n");
print META $meta;
close(META);
$parsed_meta = eval { YAML::Load($meta); };
} elsif((my $res = $ua->request(HTTP::Request->new(GET => $METAjsonURL)))->is_success()) {
warn("Fetching JSON\n");
$meta = $res->content();
open(META, ">$METAjsonfile") || die("Can't write $METAjsonfile\n");
print META $meta;
close(META);
$parsed_meta = eval { JSON::decode_json($meta); };
} else { } else {
# read from interwebnet warn("nothing!?!?\n");
my $res = $ua->request(HTTP::Request->new(GET => $METAymlURL));
if(!$res->is_success()) {
return ('!', '!');
} else {
$yaml = $res->content();
}
open(YAML, ">$cachefile") || die("Can't write $cachefile\n");
print YAML $yaml;
close(YAML);
} }
eval { $yaml = YAML::Load($yaml); }; return ('!', '!') if($@ || !defined($parsed_meta));
return ('!', '!') if($@ || !defined($yaml));

# These are for META.yml
$yaml->{requires} ||= {}; $parsed_meta->{requires} ||= {};
$yaml->{build_requires} ||= {}; $parsed_meta->{build_requires} ||= {};
$yaml->{configure_requires} ||= {}; $parsed_meta->{configure_requires} ||= {};
$yaml->{test_requires} ||= {}; $parsed_meta->{test_requires} ||= {};
return %{$yaml->{requires}}, %{$yaml->{build_requires}}, %{$yaml->{configure_requires}}, %{$yaml->{test_requires}}; # These are for META.json
$parsed_meta->{prereqs}->{runtime}->{requires} ||= {};
$parsed_meta->{prereqs}->{configure}->{requires} ||= {};
$parsed_meta->{prereqs}->{build}->{requires} ||= {};
$parsed_meta->{prereqs}->{test}->{requires} ||= {};
return
%{$parsed_meta->{requires}},
%{$parsed_meta->{build_requires}},
%{$parsed_meta->{configure_requires}},
%{$parsed_meta->{test_requires}},
%{$parsed_meta->{prereqs}->{runtime}->{requires}},
%{$parsed_meta->{prereqs}->{configure}->{requires}},
%{$parsed_meta->{prereqs}->{build}->{requires}},
%{$parsed_meta->{prereqs}->{test}->{requires}};
} }


1; 1;
36 changes: 21 additions & 15 deletions populate-cache.pl
View file
Original file line number Original file line Diff line number Diff line change
Expand Up @@ -39,24 +39,30 @@
SELECT DISTINCT(file) FROM packages SELECT DISTINCT(file) FROM packages
")}; ")};


print "Getting META.ymls\n"; print "Getting META.ymls and META.jsons\n";
foreach my $file (@files) { foreach my $file (@files) {
$file =~ m{^./../([^/]+)(/.*)?/([^/]*).(tar.gz|tgz|zip)$}; $file =~ m{^./../([^/]+)(/.*)?/([^/]*).(tar.gz|tgz|zip)$};
my($author, $dist) = ($1, $3); my($author, $dist) = ($1, $3);
next if(!defined($author) || !defined($dist)); next if(!defined($author) || !defined($dist));
my $local_file = "db/META.yml/$dist.yml";
my $remote_file = "http://search.cpan.org/src/$author/$dist/META.yml"; foreach my $tuple (

{ local => "db/META.yml/$dist.yml", remote => "http://search.cpan.org/src/$author/$dist/META.yml" },
next if(-e $local_file); { local => "db/META.yml/$dist.json", remote => "http://search.cpan.org/src/$author/$dist/META.json" },

) {
my $res = $ua->request(HTTP::Request->new(GET => $remote_file)); my $local_file = $tuple->{local};
if(!$res->is_success()) { my $remote_file = $tuple->{remote};
next;
} else { next if(-e $local_file);
my $yaml = $res->content();
open(FILE, '>', $local_file) || die("Can't write $local_file\n"); my $res = $ua->request(HTTP::Request->new(GET => $remote_file));
print FILE $yaml; if(!$res->is_success()) {
close(FILE); next;
sleep 1; } else {
my $yaml = $res->content();
open(FILE, '>', $local_file) || die("Can't write $local_file\n");
print FILE $yaml;
close(FILE);
sleep 1;
}
} }
} }
2 changes: 1 addition & 1 deletion templates/cpandeps.tt2
View file
Original file line number Original file line Diff line number Diff line change
Expand Up @@ -248,7 +248,7 @@ pageTracker._trackPageview();
<br><br>Hosted on <a href=http://www.uk2.net/dedicated-servers/>dedicated servers</a> from UK2.net <br><br>Hosted on <a href=http://www.uk2.net/dedicated-servers/>dedicated servers</a> from UK2.net


<br clear="both"> <br clear="both">
[% debug.replace("\n","<br>").replace(" ","&nbsp;") %] [% debug.replace("\n","<br>").replace(" ","&nbsp;"); %]
</div> </div>


</body> </body>
Expand Down

0 comments on commit 11b5dd4

Please sign in to comment.