Permalink
Browse files

use META.json as well as META.yml

  • Loading branch information...
1 parent 057af2b commit 11b5dd4f47612c0834ff67a78ecc8508004e42b5 @DrHyde committed May 15, 2012
Showing with 98 additions and 54 deletions.
  1. +22 −11 build-reverse-index.pl
  2. +54 −27 lib/CPANdeps.pm
  3. +21 −15 populate-cache.pl
  4. +1 −1 templates/cpandeps.tt2
@@ -6,6 +6,7 @@
use Data::Dumper;
use FindBin;
use YAML ();
+use JSON ();
$/ = undef;
@@ -26,20 +27,30 @@
mkdir 'db/reverse';
chmod 0777, qw(db db/reverse);
-my %METAyml;
+my %META;
opendir(DIR, 'db/META.yml') || die("can't open db/META.yml\n");
foreach(grep { -f "db/META.yml/$_" } readdir(DIR)) {
open FILE, "db/META.yml/$_" || die("Can't read db/META.yml/$_\n");
eval {
- $METAyml{$_} = YAML::Load(<FILE>);
- $METAyml{$_} = join("\n",
- keys %{$METAyml{$_}->{requires}},
- keys %{$METAyml{$_}->{build_requires}},
- keys %{$METAyml{$_}->{configure_requires}},
- keys %{$METAyml{$_}->{test_requires}},
- );
+ if($_ =~ /yml$/) {
+ $META{$_} = YAML::Load(<FILE>);
+ $META{$_} = join("\n",
+ keys %{$META{$_}->{requires}},
+ keys %{$META{$_}->{build_requires}},
+ keys %{$META{$_}->{configure_requires}},
+ keys %{$META{$_}->{test_requires}},
+ );
+ } else {
+ $META{$_} = JSON::decode_json(<FILE>);
+ $META{$_} = join("\n",
+ keys %{$META{$_}->{prereqs}->{runtime}->{requires}},
+ keys %{$META{$_}->{prereqs}->{configure}->{requires}},
+ keys %{$META{$_}->{prereqs}->{build}->{requires}},
+ keys %{$META{$_}->{prereqs}->{test}->{requires}},
+ );
+ }
};
- delete $METAyml{$_} if($@);
+ delete $META{$_} if($@);
close(FILE);
}
closedir(DIR);
@@ -55,8 +66,8 @@
print FILE Dumper([
sort keys %{{
map { s/\.yml$//; s/-v?\d[\d.]*$//; $_ => 1 }
- grep { $METAyml{$_} =~ /(^|\s)($modules)(\s|$)/ }
- keys %METAyml
+ grep { $META{$_} =~ /(^|\s)($modules)(\s|$)/ }
+ keys %META
}}
]);
close(FILE);
View
@@ -1,5 +1,3 @@
-# $Id: CPANdeps.pm,v 1.37 2009/02/14 23:03:53 drhyde Exp $
-
package CPANdeps;
use strict;
@@ -10,6 +8,7 @@ use Cwd;
use CGI;
use CGI::Carp qw(fatalsToBrowser);
use YAML ();
+use JSON ();
use DBI;
use LWP::UserAgent;
@@ -461,36 +460,64 @@ sub getpurity {
sub getreqs {
my($author, $distname, $ua) = @_;
- my $cachefile = "$home/db/META.yml/$distname.yml";
- my $METAymlURL = "http://search.cpan.org/src/$author/$distname/META.yml";
- my $yaml;
+ my $METAymlfile = "$home/db/META.yml/$distname.yml";
+ my $METAjsonfile = "$home/db/META.yml/$distname.json";
+ my $METAymlURL = "http://search.cpan.org/src/$author/$distname/META.yml";
+ my $METAjsonURL = "http://search.cpan.org/src/$author/$distname/META.json";
+ my $meta;
local $/ = undef;
- # if we have a cache file, read it
- if(-e $cachefile) {
- open(YAML, $cachefile) || die("Can't read $cachefile\n");
- $yaml = <YAML>;
+ my $parsed_meta;
+ if(-e $METAymlfile) {
+ warn("Cached YAML\n");
+ open(YAML, $METAymlfile) || die("Can't read $METAymlfile\n");
+ $meta = <YAML>;
close(YAML);
+ $parsed_meta = eval { YAML::Load($meta); };
+ } elsif(-e $METAjsonfile) {
+ warn("Cached JSON\n");
+ open(JSON, $METAjsonfile) || die("Can't read $METAjsonfile\n");
+ $meta = <JSON>;
+ close(JSON);
+ $parsed_meta = eval { JSON::decode_json($meta); };
+ } elsif((my $res = $ua->request(HTTP::Request->new(GET => $METAymlURL)))->is_success()) {
+ warn("Fetching YAML\n");
+ $meta = $res->content();
+ open(META, ">$METAymlfile") || die("Can't write $METAymlfile\n");
+ print META $meta;
+ close(META);
+ $parsed_meta = eval { YAML::Load($meta); };
+ } elsif((my $res = $ua->request(HTTP::Request->new(GET => $METAjsonURL)))->is_success()) {
+ warn("Fetching JSON\n");
+ $meta = $res->content();
+ open(META, ">$METAjsonfile") || die("Can't write $METAjsonfile\n");
+ print META $meta;
+ close(META);
+ $parsed_meta = eval { JSON::decode_json($meta); };
} else {
- # read from interwebnet
- my $res = $ua->request(HTTP::Request->new(GET => $METAymlURL));
- if(!$res->is_success()) {
- return ('!', '!');
- } else {
- $yaml = $res->content();
- }
- open(YAML, ">$cachefile") || die("Can't write $cachefile\n");
- print YAML $yaml;
- close(YAML);
+ warn("nothing!?!?\n");
}
- eval { $yaml = YAML::Load($yaml); };
- return ('!', '!') if($@ || !defined($yaml));
-
- $yaml->{requires} ||= {};
- $yaml->{build_requires} ||= {};
- $yaml->{configure_requires} ||= {};
- $yaml->{test_requires} ||= {};
- return %{$yaml->{requires}}, %{$yaml->{build_requires}}, %{$yaml->{configure_requires}}, %{$yaml->{test_requires}};
+ return ('!', '!') if($@ || !defined($parsed_meta));
+
+ # These are for META.yml
+ $parsed_meta->{requires} ||= {};
+ $parsed_meta->{build_requires} ||= {};
+ $parsed_meta->{configure_requires} ||= {};
+ $parsed_meta->{test_requires} ||= {};
+ # These are for META.json
+ $parsed_meta->{prereqs}->{runtime}->{requires} ||= {};
+ $parsed_meta->{prereqs}->{configure}->{requires} ||= {};
+ $parsed_meta->{prereqs}->{build}->{requires} ||= {};
+ $parsed_meta->{prereqs}->{test}->{requires} ||= {};
+ return
+ %{$parsed_meta->{requires}},
+ %{$parsed_meta->{build_requires}},
+ %{$parsed_meta->{configure_requires}},
+ %{$parsed_meta->{test_requires}},
+ %{$parsed_meta->{prereqs}->{runtime}->{requires}},
+ %{$parsed_meta->{prereqs}->{configure}->{requires}},
+ %{$parsed_meta->{prereqs}->{build}->{requires}},
+ %{$parsed_meta->{prereqs}->{test}->{requires}};
}
1;
View
@@ -39,24 +39,30 @@
SELECT DISTINCT(file) FROM packages
")};
-print "Getting META.ymls\n";
+print "Getting META.ymls and META.jsons\n";
foreach my $file (@files) {
$file =~ m{^./../([^/]+)(/.*)?/([^/]*).(tar.gz|tgz|zip)$};
my($author, $dist) = ($1, $3);
next if(!defined($author) || !defined($dist));
- my $local_file = "db/META.yml/$dist.yml";
- my $remote_file = "http://search.cpan.org/src/$author/$dist/META.yml";
-
- next if(-e $local_file);
-
- my $res = $ua->request(HTTP::Request->new(GET => $remote_file));
- if(!$res->is_success()) {
- next;
- } else {
- my $yaml = $res->content();
- open(FILE, '>', $local_file) || die("Can't write $local_file\n");
- print FILE $yaml;
- close(FILE);
- sleep 1;
+
+ foreach my $tuple (
+ { local => "db/META.yml/$dist.yml", remote => "http://search.cpan.org/src/$author/$dist/META.yml" },
+ { local => "db/META.yml/$dist.json", remote => "http://search.cpan.org/src/$author/$dist/META.json" },
+ ) {
+ my $local_file = $tuple->{local};
+ my $remote_file = $tuple->{remote};
+
+ next if(-e $local_file);
+
+ my $res = $ua->request(HTTP::Request->new(GET => $remote_file));
+ if(!$res->is_success()) {
+ next;
+ } else {
+ my $yaml = $res->content();
+ open(FILE, '>', $local_file) || die("Can't write $local_file\n");
+ print FILE $yaml;
+ close(FILE);
+ sleep 1;
+ }
}
}
@@ -248,7 +248,7 @@ pageTracker._trackPageview();
<br><br>Hosted on <a href=http://www.uk2.net/dedicated-servers/>dedicated servers</a> from UK2.net
<br clear="both">
- [% debug.replace("\n","<br>").replace(" ","&nbsp;") %]
+ [% debug.replace("\n","<br>").replace(" ","&nbsp;"); %]
</div>
</body>

0 comments on commit 11b5dd4

Please sign in to comment.