Merge branch 'master' of github.com:CPAN-API/cpan-api

trapd00r · Jan 6, 2011 · 1d02112 · 1d02112
2 parents 30c12d4 + e5668f3
commit 1d02112
Show file tree

Hide file tree

Showing 6 changed files with 169 additions and 33 deletions.
diff --git a/elasticsearch/index_dists.pl b/elasticsearch/index_dists.pl
@@ -24,7 +24,8 @@
 
 if ( $cpan->dist_like ) {
     say "searching for dists like: " . $cpan->dist_like;
-    $dists = search_dists( { dist => { like => $cpan->dist_like, '!=' => undef, } } );
+    $dists = search_dists(
+        { dist => { like => $cpan->dist_like, '!=' => undef, } } );
 }
 
 elsif ( $cpan->dist_name ) {
@@ -37,7 +38,9 @@
     $dists = search_dists();
 }
 
-foreach my $dist ( @{$dists} ) {
+my %reverse = reverse %{$dists};
+
+foreach my $dist ( sort values %{$dists} ) {
     process_dist( $dist );
 }
 
@@ -51,22 +54,28 @@ sub process_dist {
 
     say '+' x 20 . " DIST: $distvname" if $cpan->debug;
 
-    my $dist = MetaCPAN::Dist->new( distvname => $distvname, module_rs => $cpan->module_rs );
+    my $dist = MetaCPAN::Dist->new(
+        distvname => $distvname,
+        dist_name => $reverse{$distvname},
+        module_rs => $cpan->module_rs
+    );
+
     $dist->process;
 
     say "Found " . scalar @{ $dist->processed } . " modules in dist";
-    $dist->tar->clear if $dist->tar;
+
+    #$dist->tar->clear if $dist->tar;
     $dist = undef;
-    
+
     ++$attempts;
-    
+
     # diagnostics
     if ( every( $every ) ) {
 
         my $iter_time = tv_interval( $t0,      [gettimeofday] );
         my $elapsed   = tv_interval( $t_begin, [gettimeofday] );
         say '#' x 78;
-    
+
         say "$distvname";    # if $icpan->debug;
         say "$iter_time to process dist";
         say "$elapsed so far... ($attempts dists out of $total_dists)";
@@ -81,7 +90,6 @@ sub process_dist {
 
     }
 
-
     return;
 
 }
@@ -92,23 +100,19 @@ sub search_dists {
 
     my $search = $cpan->module_rs->search(
         $constraints,
-        {   +select  => ['distvname', 'dist',],
+        {   +select  => [ 'distvname', 'dist', ],
             distinct => 1,
             order_by => 'distvname ASC',
         }
     );
-      
-    my %dist = ( );
+
+    my %dist = ();
     while ( my $row = $search->next ) {
         $dist{ $row->dist } = $row->distvname;
     }
+    say "found " . scalar (keys %dist) . " dists";
 
-    my @dists = sort values %dist;
-
-    $total_dists = scalar @dists;
-    say "found $total_dists distros";
-
-    return \@dists;
+    return \%dist;
 
 }
 
diff --git a/elasticsearch/map_modules.pl b/elasticsearch/map_modules.pl
@@ -30,6 +30,7 @@ sub put_mapping {
 
         #_source => { compress => 1 },
         properties => {
+            abstract       => { type => "string" },
             archive        => { type => "string" },
             author         => { type => "string" },
             distname       => { type => "string" },

diff --git a/elasticsearch/start_fresh.sh b/elasticsearch/start_fresh.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+perl delete_index.pl cpan
+perl create_index.pl cpan
+perl map_modules.pl
+perl index_authors.pl
+perl index_cpanratings.pl
+perl index_dists.pl --refresh_db
+#cd /home/cpan/elasticsearch && zip -r /home/cpan/data_snapshot.zip data
diff --git a/lib/MetaCPAN/Author.pm b/lib/MetaCPAN/Author.pm
@@ -100,7 +100,11 @@ sub author_config {
     # uncomment this when search.metacpan can deal with lists in values
     my @lists = qw( website email books blog_url blog_feed cats dogs );
     foreach my $key ( @lists ) {
-        if ( exists $conf->{$key} && reftype( $conf->{$key} ) ne 'ARRAY' ) {
+        if (exists $conf->{$key}
+            && (  !reftype( $conf->{$key} )
+                || reftype( $conf->{$key} ) ne 'ARRAY' )
+            )
+        {
             $conf->{$key} = [ $conf->{$key} ];
         }
     }
@@ -109,6 +113,8 @@ sub author_config {
 
 }
 
+
+
 sub _build_author_fh {
 
     my $self = shift;