Ensures that only the very latest package for a given dist is added t…

…o the main index.
patspam · Dec 18, 2010 · db050b1 · db050b1
1 parent 7070733
commit db050b1
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 22 deletions.
diff --git a/elasticsearch/index_dists.pl b/elasticsearch/index_dists.pl
@@ -17,27 +17,27 @@
 
 $cpan->debug( $ENV{'DEBUG'} );
 
-my @dists = ();
+my $dists = [];
 
 my $total_dists = 1;
 say $cpan->dist_name;
 
 if ( $cpan->dist_like ) {
     say "searching for dists like: " . $cpan->dist_like;
-    @dists = search_dists( { dist => { like => $cpan->dist_like, '!=' => undef, } } );
+    $dists = search_dists( { dist => { like => $cpan->dist_like, '!=' => undef, } } );
 }
 
 elsif ( $cpan->dist_name ) {
     say "searching for dist: " . $cpan->dist_name;
-    @dists = search_dists( { dist => $cpan->dist_name } );
+    $dists = search_dists( { dist => $cpan->dist_name } );
 }
 
 else {
     say "search all dists";
-    @dists = search_dists();
+    $dists = search_dists();
 }
-say dump( \@dists);
-foreach my $dist ( @dists ) {
+
+foreach my $dist ( @{$dists} ) {
     process_dist( $dist );
 }
 
@@ -92,22 +92,23 @@ sub search_dists {
 
     my $search = $cpan->module_rs->search(
         $constraints,
-        {   +select  => 'distvname',
-            columns  => ['dist'],
+        {   +select  => ['distvname', 'dist',],
             distinct => 1,
-            order_by => 'distvname DESC',
-            rows     => 1
+            order_by => 'distvname ASC',
         }
     );
-
-    my @dists = ();
+      
+    my %dist = ( );
     while ( my $row = $search->next ) {
-        push @dists, $row->distvname;
+        $dist{ $row->dist } = $row->distvname;
     }
 
+    my @dists = sort values %dist;
+
+    $total_dists = scalar @dists;
     say "found $total_dists distros";
 
-    return @dists;
+    return \@dists;
 
 }
 
diff --git a/elasticsearch/map_modules.pl b/elasticsearch/map_modules.pl
@@ -9,23 +9,21 @@ =head1 SYNOPSIS
 use Modern::Perl;
 
 use Data::Dump qw( dump );
-use Find::Lib '../lib', '../../iCPAN/perl/lib';
+use Find::Lib '../lib';
 use MetaCPAN;
 
 my $metacpan = MetaCPAN->new();
 my $es       = $metacpan->es;
 
 put_mapping();
 
-
 sub put_mapping {
 
     $es->delete_mapping(
         index => ['cpan'],
         type  => 'module',
     );
 
-#exit(0);
     my $result = $es->put_mapping(
         index => ['cpan'],
         type  => 'module',
@@ -35,7 +33,6 @@ sub put_mapping {
             archive        => { type => "string" },
             author         => { type => "string" },
             distname       => { type => "string" },
-            distsearchname => { type => "string", index => "not_analyzed" },
             distvname      => { type => "string" },
             download_url   => { type => "string" },
             name           => { type => "string" },
@@ -45,7 +42,5 @@ sub put_mapping {
         }
     );
 
-    say dump $result;
-
 }
 
diff --git a/lib/MetaCPAN/Dist.pm b/lib/MetaCPAN/Dist.pm
@@ -88,7 +88,6 @@ sub process {
 
     my $self    = shift;
     my $success = 0;
-
     my $module_rs = $self->module_rs->search({ distvname => $self->distvname });
 
     my @modules = ();
@@ -131,6 +130,7 @@ MODULE:
 
     if ( $self->es_inserts ) {
         my $result = $self->es->bulk( $self->es_inserts );
+        #say dump( $self->es_inserts );
     }
 
     elsif ( $self->debug ) {
@@ -266,7 +266,6 @@ sub parse_pod {
     # if this line is uncommented some pod, like Dancer docs gets skipped
     delete $self->files->{$file};
     push @{$self->processed}, $file;
-    #say '9'x70 . $self->files->{$filename};
 
     return 1;