Improve speed of front page ‘recent problems’ query

CHANGELOG.md

@@ -17,6 +17,7 @@
         - Add label to questionnaire textarea. #3944
         - Offline report drafting. #4290
         - Set width=device-width in viewport meta tag. #4384
+        - Improve performance of front page ‘recent problems’ query. #4424
     - Bugfixes:
         - Add ID attributes to change password form inputs.
         - Fix link deactivation for privacy policy link on privacy policy page. #3704

perllib/FixMyStreet/DB/ResultSet/Problem.pm

@@ -162,7 +162,10 @@ sub _recent {
     $query->{photo} = { '!=', undef } if $photos;
 
     my $attrs = {
-        order_by => { -desc => \'coalesce(confirmed, created)' },
+        # We order by most recently _created_, not confirmed, as the latter
+        # is too slow on installations with millions of reports.
+        # The correct ordering is applied by the `sort` lines below.
+        order_by => { -desc => 'id' },
         rows => $num,
     };
 
@@ -171,25 +174,36 @@ sub _recent {
         $attrs->{bind} = [ $lat, $lon, $dist ];
         $attrs->{join} = 'nearby';
         $probs = [ mySociety::Locale::in_gb_locale {
-            $rs->search( $query, $attrs )->all;
+            sort { _cmp_reports($b, $a) } $rs->search( $query, $attrs )->all;
         } ];
     } else {
-        $probs = Memcached::get($key);
+        $probs = Memcached::get($key) unless FixMyStreet->test_mode;
         if ($probs) {
             # Need to refetch to check if hidden since cached
-            $probs = [ $rs->search({
+            $probs = [ sort { _cmp_reports($b, $a) } $rs->search({
                 id => [ map { $_->id } @$probs ],
                 %$query,
             }, $attrs)->all ];
         } else {
-            $probs = [ $rs->search( $query, $attrs )->all ];
+            $probs = [ sort { _cmp_reports($b, $a) } $rs->search( $query, $attrs )->all ];
             Memcached::set($key, $probs, _cache_timeout());
         }
     }
 
     return $probs;
 }
 
+sub _cmp_reports {
+    my ($a, $b) = @_;
+
+    # reports may not be confirmed
+    my $a_confirmed = $a->confirmed ? $a->confirmed->epoch : 0;
+    my $b_confirmed = $b->confirmed ? $b->confirmed->epoch : 0;
+
+    return $a_confirmed <=> $b_confirmed;
+}
+
+
 # Problems around a location
 
 sub around_map {

t/app/controller/index.t

@@ -1,4 +1,7 @@
 use FixMyStreet::TestMech;
+use DateTime;
+use Web::Scraper;
+
 my $mech = FixMyStreet::TestMech->new;
 
 use t::Mock::Nominatim;
@@ -100,6 +103,27 @@ subtest "prefilters /around if filter_category given in URL" => sub {
     $mech->content_contains("MyUniqueTestGroup");
 };
 
+subtest "recent reports are correctly shown on front page" => sub {
+    FixMyStreet::DB->resultset('Problem')->delete_all;
+
+    my ($r1, $r2, $r3)= $mech->create_problems_for_body(3, 2651, 'Recent problem');
+
+    # Make the oldest problem have the newest confirmed timestamp so it appears top of recent list
+    $r1->update( { confirmed => $r1->confirmed + DateTime::Duration->new( minutes => 5 ) } );
+
+    $mech->get_ok('/');
+
+    my $result = scraper {
+        process '#front-recently ul li h3', 'title[]', 'TEXT';
+    }->scrape( $mech->response );
+
+    is_deeply $result->{title}, [
+        'Recent problem Test 3 for 2651',
+        'Recent problem Test 1 for 2651',
+        'Recent problem Test 2 for 2651'
+    ], "correct ordering of recent problems";
+};
+
 END {
     done_testing();
 }