Process MetaCPAN lookup in chunks

Fixes #47.
tokuhirom · Jan 8, 2017 · 1238e20 · 1238e20
1 parent f108d71
commit 1238e20
Show file tree

Hide file tree

Showing 2 changed files with 69 additions and 24 deletions.
diff --git a/lib/OrePAN2/Indexer.pm b/lib/OrePAN2/Indexer.pm
@@ -33,6 +33,8 @@ sub new {
 
 sub directory { shift->{directory} }
 
+sub metacpan_lookup_size { shift->{metacpan_lookup_size} || 200 }
+
 sub make_index {
     my ( $self, %args ) = @_;
 
@@ -155,36 +157,40 @@ sub do_metacpan_lookup {
     my $mc = MetaCPAN::Client->new( version => 'v1' );
     my @archives = map { Path::Tiny->new($_)->basename } @{$files};
     my @search_by_archives = map { +{ archive => $_ } } @archives;
-    my $releases = $mc->release( { either => \@search_by_archives } );
 
-    my @file_search;
+    while (@search_by_archives) {
+        my @search_by_archives_chunk
+            = splice @search_by_archives, 0, $self->metacpan_lookup_size;
 
-    while ( my $release = $releases->next ) {
+        my $releases = $mc->release( { either => \@search_by_archives_chunk } );
 
-        $provides->{archive}->{ $release->archive } = $release->name;
+        my @file_search;
 
-        push @file_search,
-            {
-            all => [
-                { release          => $release->name },
-                { indexed          => 'true' },
-                { authorized       => 'true' },
-                { 'module.indexed' => 'true' },
-            ]
-            };
-    }
+        while ( my $release = $releases->next ) {
+            $provides->{archive}->{ $release->archive } = $release->name;
 
-    return unless @file_search;
+            push @file_search,
+                {
+                    all => [
+                        { release          => $release->name },
+                        { indexed          => 'true' },
+                        { authorized       => 'true' },
+                        { 'module.indexed' => 'true' },
+                    ]
+                };
+        }
 
-    my $modules = $mc->module( { either => \@file_search } );
+        next unless @file_search;
 
-    while ( my $file = $modules->next ) {
-        my $module = $file->module or next;
-        foreach my $inner ( is_arrayref $module ? @{$module} : $module ) {
-            next unless $inner->{indexed};
+        my $modules = $mc->module( { either => \@file_search } );
 
-            $provides->{release}->{ $file->release }->{ $inner->{name} } //=
-                $inner->{version};
+        while ( my $file = $modules->next ) {
+            my $module = $file->module or next;
+            for my $inner ( is_arrayref $module ? @{$module} : $module ) {
+                next unless $inner->{indexed};
+                $provides->{release}->{ $file->release }->{ $inner->{name} } //=
+                    $inner->{version};
+            }
         }
     }
 

diff --git a/t/06_inject_live.t b/t/06_inject_live.t
@@ -44,8 +44,7 @@ subtest 'use MetaCPAN' => sub {
 
     my $release = inject_module( 'OrePAN2', $tmpdir );
 
-    my $path = $release->download_url;
-    $path =~ s{\A.*/authors/}{};
+    my $path = url2path( $release->download_url );
 
     ok -f "$tmpdir/authors/$path", 'path exists';
 
@@ -66,6 +65,40 @@ subtest 'use MetaCPAN' => sub {
     );
 };
 
+subtest 'MetaCPAN lookup works in chunks' => sub {
+    my $tmpdir = tempdir( CLEANUP => 1 );
+    my @modules = qw( OrePAN2 autodie );
+    my %release;
+
+    for my $module ( @modules ) {
+        my $release = inject_module( $module, $tmpdir );
+        $release{$module}{$_} = $release->$_
+            for qw( download_url archive name );
+    }
+
+    my $orepan = OrePAN2::Indexer->new(
+        directory => $tmpdir,
+        metacpan => 1,
+        metacpan_lookup_size => 1,
+    );
+
+    $orepan->do_metacpan_lookup( [
+        map url2path( $release{$_}{download_url} ), @modules
+    ] );
+
+    for my $module ( @modules ) {
+        ok(
+            exists $orepan->_metacpan_lookup
+                ->{archive}{ $release{$module}{archive} },
+            "%module archive found by MetaCPAN"
+        );
+        ok(
+            $orepan->_metacpan_lookup->{release}{ $release{$module}{name} },
+            "$module release found by MetaCPAN"
+        );
+    }
+};
+
 subtest 'Upgrade undef versions' => sub {
     my $tmpdir = tempdir( CLEANUP => 1 );
 
@@ -124,4 +157,10 @@ sub inject_and_index {
     return $orepan->make_index;
 }
 
+sub url2path {
+    my ( $url ) = @_;
+    ( my $path = $url ) =~ s{\A.*/authors/}{};
+    return $path
+}
+
 done_testing;