Combine results in the same file and collapse them

Also show the last time the index was updated.
dgl · Mar 31, 2012 · b52de45 · b52de45
1 parent fa77de2
commit b52de45
Show file tree

Hide file tree

Showing 6 changed files with 127 additions and 62 deletions.
diff --git a/lib/WWW/CPANGrep.pm b/lib/WWW/CPANGrep.pm
@@ -28,7 +28,7 @@ sub dispatch_request {
 
       return [ 200, ['Content-type' => 'application/json' ],
                [ encode_json({
-                   count => scalar @{$r->{results}},
+                   count => $r->{count},
                    duration => $r->{duration},
                    results => [grep defined, @{$r->{results}}[0 .. $limit]]
                  })]
@@ -47,7 +47,7 @@ sub dispatch_request {
     my $r = $self->_search($q);
     # XXX: Urgh, stop abusing render_response for everything like this...
     if(ref $r eq 'HASH') {
-      return render_response($q, $r->{results}, "", $r->{duration}, $page_number);
+      return render_response($q, $r->{results}, "", $r->{duration}, $page_number, $r->{count});
     } else {
       return render_response($q, undef, $r, undef);
     }
@@ -56,7 +56,11 @@ sub dispatch_request {
     HTML::Zoom->from_file(TMPL_PATH . "/about.html")
   },
   sub (/) {
+    # XXX: Fix me if this ever becomes an overhead
+    my $redis = AnyEvent::Redis->new(host => $config->{"server.queue"});
     HTML::Zoom->from_file(TMPL_PATH . "/grep.html")
+      ->select('#lastupdate')
+      ->replace_content($redis->get("cpangrep:lastindex")->recv);
   },
 };
 
@@ -69,31 +73,28 @@ sub _search {
   my $redis = AnyEvent::Redis->new(host => $config->{"server.queue"});
   my $start = AE::time;
 
-  my $results;
-  my $response;
+  my %res;
   my $cache = $redis->get("querycache:" . uri_escape_utf8($q))->recv;
   if(!$ENV{DEBUG} && $cache) {
-    $results = decode_json($cache);
+    %res = %{decode_json($cache)};
   } else {
-    my %res = $search->search($redis);
+    %res = $search->search($redis);
     if($res{error}) {
-      $response = "Something went wrong! $res{error}";
-      return $response;
+      return "Something went wrong! $res{error}";
     } else {
       my $redis_cache = AnyEvent::Redis->new(host => $config->{"server.queue"});
-      $redis_cache->setex("querycache:" . uri_escape_utf8($q), 1800, encode_json($res{results}))->recv;
-      $results = $res{results};
+      $redis_cache->setex("querycache:" . uri_escape_utf8($q), 1800, encode_json(\%res))->recv;
     }
   }
 
   my $duration = AE::time - $start;
   printf "Took %0.2f %s\n", $duration, $cache ? "(cached)" : "";
 
-  return { results => $results, duration => $duration };
+  return { results => $res{results}, duration => $duration, count => $res{count} };
 }
 
 sub render_response {
-  my($q, $results, $error, $duration, $page_number) = @_;
+  my($q, $results, $error, $duration, $page_number, $count) = @_;
 
   my $pager = Data::Pageset::Render->new({
       total_entries    => $results ? scalar @$results : 0,
@@ -106,7 +107,7 @@ sub render_response {
 
   my $output = HTML::Zoom->from_file(TMPL_PATH . "/results.html")
     ->select('title')->replace_content("$q · CPAN->grep")
-    ->select('#total')->replace_content(!$results || @$results > MAX ? "more than " . MAX : scalar @$results)
+    ->select('#total')->replace_content($count > MAX ? "more than " . MAX : $count)
     ->select('#time')->replace_content(sprintf "%0.2f", $duration || 0)
     ->select('#start-at')->replace_content($pager->first)
     ->select('#end-at')->replace_content($pager->last)
@@ -119,35 +120,48 @@ sub render_response {
   } else {
     $output = $output->select('.results')->repeat_content(
       [ map {
-        my $i = $_;
-        my $result = $results->[$i];
+        my $result = $_;
         sub {
-          my($package) = $result->{file}->{dist} =~ m{([^/]+)$};
+          my($package) = $result->{dist} =~ m{([^/]+)$};
           $package =~ s/\.(?:tar\.gz|zip|tar\.bz2)$//;
-          my $file = "$package/$result->{file}->{file}";
-          my $author = ($result->{file}->{dist} =~ m{^([^/]+)})[0];
-
-          # XXX: Find some better code for this.
-          my $html = eval { my $html = "";
-            $html .= encode_entities(substr $result->{text}, 0, $result->{match}->[0]) if $result->{match}->[0];
-            $html .= "<strong>";
-            $html .= encode_entities(substr $result->{text}, $result->{match}->[0], $result->{match}->[1] - $result->{match}->[0]);
-            $html .= "</strong>";
-            $html .= encode_entities(substr $result->{text}, $result->{match}->[1]);
-            $html;
-          } or do print "$@";
-          $html ||= "";
-
-          $_->select('.file-link')->replace_content($file)
-          ->then
-          # TODO: Use metacpan here.
-          ->set_attribute(href => "http://cpansearch.perl.org/src/$author/$file")
-          ->select('.dist-link')->replace_content("$author/$package")
-          ->then
-          ->set_attribute(href => "http://search.cpan.org/~" . lc($author) . "/$package/")
-          ->select('.excerpt')->replace_content(\$html);
+          my $author = ($result->{dist} =~ m{^([^/]+)})[0];
+
+          use Data::Dump qw(pp); pp $result;
+
+          $_ = $_->select('.files')->repeat_content([map {
+            my $file = $_;
+            sub {
+              $_ = $_->select('.excerpts')->repeat_content([map {
+                my $excerpt = $_;
+                sub {
+                  # XXX: Find some better code for this.
+                  my $html = eval { my $html = "";
+                    $html .= encode_entities(substr $excerpt->{text}, 0, $excerpt->{match}->[0]) if $excerpt->{match}->[0];
+                    $html .= "<strong>";
+                    $html .= encode_entities(substr $excerpt->{text}, $excerpt->{match}->[0], $excerpt->{match}->[1] - $excerpt->{match}->[0]);
+                    $html .= "</strong>";
+                    $html .= encode_entities(substr $excerpt->{text}, $excerpt->{match}->[1]);
+                    $html;
+                  } or do print "$@";
+                  $html ||= "";
+                  $_->select('.excerpt')->replace_content(\$html);
+                }
+              } @{$file->{results}}]);
+
+              my $filename = "$package/$file->{file}";
+              $_->select('.file-link')->replace_content($filename)
+                ->then
+                # TODO: Use metacpan here.
+                ->set_attribute(href => "http://cpansearch.perl.org/src/$author/$filename");
+            }
+          } @{$result->{files}}]);
+
+          $_->select('.dist-link')->replace_content("$author/$package")
+            ->then
+            # TODO: Use metacpan here.
+            ->set_attribute(href => "http://search.cpan.org/~" . lc($author) . "/$package/");
         }
-      } ($pager->first - 1) .. ($pager->last - 1)]);
+      } @$results[$pager->first - 1 .. $pager->last - 1]]);
 
     $output = $output->select('.pagination')->replace_content(\$pager->html);
   }

diff --git a/lib/WWW/CPANGrep/Index.pm b/lib/WWW/CPANGrep/Index.pm
@@ -56,12 +56,25 @@ sub index {
 
   delete $self->redis->{"new-index"};
 
-  WWW::CPANGrep::Index::Worker->new(
+  my $done = WWW::CPANGrep::Index::Worker->new(
     cpan_dir => $self->cpan_dir,
     slab_dir => $self->slab_dir,
     redis_server => $self->redis_server,
     jobs => $self->jobs,
   )->run($queue);
+
+  if($done) {
+    my $redis_conn = (tied %{$self->redis})->{_conn};
+    eval { $redis_conn->rename("cpangrep:slabs", "cpangrep:slabs-old") };
+    $redis_conn->rename("new-index", "cpangrep:slabs");
+    $redis_conn->save;
+
+    for my $slab(@{$self->redis->{"cpangrep:slabs-old"}}) {
+      unlink $self->slab_dir . "/" . $slab;
+    }
+
+    $self->redis->{"cpangrep:lastindex"} = $packages->last_updated;
+  }
 }
 
 __PACKAGE__->meta->make_immutable;

diff --git a/lib/WWW/CPANGrep/Index/Worker.pm b/lib/WWW/CPANGrep/Index/Worker.pm
@@ -78,20 +78,11 @@ sub run {
   }
 
   my $name = $self->_slab->finish;
-
   # Tie::Redis currently won't autovivify :(
   $self->redis->{"new-index"} ||= [];
   push @{$self->redis->{"new-index"}}, @{$self->redis->{$name}};
 
-  if($redis_conn->decr("cpangrep:indexer") == 0) {
-    eval { $redis_conn->rename("cpangrep:slabs", "cpangrep:slabs-old") };
-    $redis_conn->rename("new-index", "cpangrep:slabs");
-    $redis_conn->save;
-
-    for my $slab(@{$self->redis->{"cpangrep:slabs-old"}}) {
-      unlink $self->slab_dir . "/" . $slab;
-    }
-  }
+  return $redis_conn->decr("cpangrep:indexer") == 0;
 }
 
 sub index_dist {

diff --git a/lib/WWW/CPANGrep/Search.pm b/lib/WWW/CPANGrep/Search.pm
@@ -211,15 +211,15 @@ sub search {
   }
 
   my @finish = $other_cv->recv;
-  return results => $self->filter_results(\@results), @finish;
+  return $self->filter_results(\@results), @finish;
 }
 
+# This could probably be optimised a lot, but take the lazy approach for now.
 sub filter_results {
   my($self, $results) = @_;
   my @results = @$results;
 
   for my $option(@{$self->_options}) {
-    # This could probably be optimised a lot, but take the lazy approach for now.
     my $predicate = 
       $option->{type} eq 'file'   ? sub { $_->{file}->{file} =~ $option->{re} } :
       $option->{type} eq 'dist'   ? sub { CPAN::DistnameInfo->new($_->{file}->{dist})->dist =~ $option->{re} } :
@@ -230,7 +230,45 @@ sub filter_results {
     @results = grep $matcher->(), @results;
   }
 
-  return \@results;
+  my $count = scalar @results;
+
+  # This could probably do with some work, currently ordering by dist with the most matches.
+  # Would maybe make sense to do per file counts, weight t/, etc less, and so on.
+  my %dists;
+  for my $result(@results) {
+    push @{$dists{$result->{file}->{dist}}}, $result;
+  }
+
+  my @ordered;
+  for my $dist(sort { @{$dists{$b}} <=> @{$dists{$a}} } keys %dists) {
+    my @dist_results = @{$dists{$dist}};
+
+    my %files;
+    for my $result(@dist_results) {
+      push @{$files{$result->{file}->{file}}}, $result;
+    }
+
+    push @ordered, { dist => $dist, files => [], truncated => 0 };
+
+    my $i = 0;
+    for my $file(sort { lc $a cmp lc $b } keys %files) {
+      my @file_results = @{$files{$file}};
+
+      if(@file_results > 3 && keys %files > 1) {
+        @file_results = @file_results[0 .. 2];
+        $file_results[-1]->{truncated} = 1;
+      }
+
+      push $ordered[-1]->{files}, { file => $file, results => \@file_results };
+
+      if($i++ >= 20 / keys %dists) {
+        $ordered[-1]->{truncated} = 1;
+        last;
+      }
+    }
+  }
+
+  return results => \@ordered, count => $count;
 }
 
 1;
diff --git a/share/html/grep.html b/share/html/grep.html
@@ -10,7 +10,15 @@
 	<meta name="robots" content="all" />
         <link rel="icon" type="image/png" href="/favicon.png" />
         <script src="http://ajax.googleapis.com/ajax/libs/jquery/1.7.1/jquery.min.js"></script>
-	<link rel="stylesheet" type="text/css" media="screen, projection" href="//dg.cx/cpangrep/cpangrep.css" />
+        <link rel="stylesheet" type="text/css" media="screen, projection" href="//dg.cx/cpangrep/cpangrep.css" />
+        <style>
+          .footer {
+            position: absolute;
+            top: 90%;
+            width: 98%;
+            text-align: center;
+          }
+        </style>
 </head>
 
 <body onload="document.forms[0].q.focus();">
@@ -30,8 +38,8 @@
 		</form>
 
 		<div class="footer">
-		<a href="about.html">About</a> &mdash; <a href="https://plus.google.com/112646864076593101108/posts">Status/updates on Google+</a>
-		<span style="position: absolute; right: 10px; bottom: 10px;"><a href="//bigv.io"><img src="//s.cpan.me/bigv_poweredby_landscape_143_x_49.png" border="0"></span>
+                  <a href="about.html">About</a> &mdash; <a href="https://plus.google.com/112646864076593101108/posts">Status/updates on Google+</a><br/>
+                  <small>Index last updated: <span id="lastupdate"></span></small>
 		</div>
 
 	</div>

diff --git a/share/html/results.html b/share/html/results.html
@@ -41,15 +41,16 @@
 	</div>
 
 	<div class="results">
-
-		<div class="result">
-
-			<a href="blah" class="file-link">Bio-LITE-Taxonomy-NCBI-0.04/t/data/names.dmp</a>
+                <div class="result">
+                  <span class="files"><span class="file">
+                      <a href="blah" class="file-link">Bio-LITE-Taxonomy-NCBI-0.04/t/data/names.dmp</a>
+                      <span class="excerpts">
 			<pre class="excerpt">52471	|	Ferula assa-foetida L.	|		|	synonym	|
 52471	|	asafoetida	|		|	common name	|
 52473	|	Astrantia epipactis	|		|	synonym	|</pre>
-			<a href="bleh" class="dist-link">MOTIF/Bio-LITE-Taxonomy-NCBI-0.04.tar.gz</a>
-
+                      </span></span>
+                   </span>
+                   <a href="bleh" class="dist-link">MOTIF/Bio-LITE-Taxonomy-NCBI-0.04.tar.gz</a>
 		</div>
 
 	</div>