Skip invalidating yesterday if a proper archive is already in progress

core/CronArchive.php

@@ -27,6 +27,7 @@
 use Piwik\DataAccess\Model;
 use Piwik\Exception\UnexpectedWebsiteFoundException;
 use Piwik\Metrics\Formatter;
+use Piwik\Period\Day;
 use Piwik\Period\Factory as PeriodFactory;
 use Piwik\CronArchive\SegmentArchiving;
 use Piwik\Period\Range;
@@ -914,6 +915,24 @@ public function invalidateRecentDate($dateStr, $idSite)
             return;
         }
 
+        $isYesterday = $dateStr === 'yesterday';
+        if ($isYesterday) {
+            // Skip invalidation for yesterday if archiving for yesterday was already started after midnight in site's timezone
+            $invalidationsInProgress = $this->model->getInvalidationsInProgress($idSite);
+            $today = Date::factoryInTimezone('today', $timezone);
+
+            foreach ($invalidationsInProgress as $invalidation) {
+                if (
+                    $invalidation['period'] == Day::PERIOD_ID
+                    && $date->toString() === $invalidation['date1']
+                    && Date::factory($invalidation['ts_started'], $timezone)->getTimestamp() >= $today->getTimestamp()
+                ) {
+                    $this->logger->debug("  " . ucfirst($dateStr) . " archive already in process for idSite = $idSite, skipping invalidation...");
+                    return;
+                }
+            }
+        }
+
         $this->logger->info("  Will invalidate archived reports for $dateStr in site ID = {idSite}'s timezone ({date}).", [
             'idSite' => $idSite,
             'date' => $date->getDatetime(),
@@ -922,7 +941,6 @@ public function invalidateRecentDate($dateStr, $idSite)
         // if we are invalidating yesterday here, we are only interested in checking if there is no archive for yesterday, or the day has changed since
         // the last archive was archived (in which there may have been more visits before midnight). so we disable the ttl check, since any archive
         // will be good enough, if the date hasn't changed.
-        $isYesterday = $dateStr == 'yesterday';
         $this->invalidateWithSegments([$idSite], $date->toString(), 'day', false, $doNotIncludeTtlInExistingArchiveCheck = $isYesterday);
     }
 

tests/PHPUnit/Integration/CronArchiveTest.php

@@ -463,6 +463,89 @@ public function getInvalidations()
         return $mock;
     }
 
+    /**
+     * @dataProvider getInvalidateYesterdayTestData
+     */
+    public function testInvalidateRecentDateForYesterdayIsSkippedWhenAlreadyInProgress($timezone, $tsStarted, $expectedInvalidationCalls)
+    {
+        $idSite = Fixture::createWebsite('2019-04-04 03:45:45', 0, false, false, 1, null, null, $timezone);
+
+        $offset = Date::getUtcOffset($timezone);
+        Date::$now = Date::factory('2020-02-03 04:05:06')->subSeconds($offset)->getTimestamp();
+
+        $this->insertInvalidations([
+            [
+                'idarchive' => 1,
+                'idsite' => 1,
+                'period' => 1,
+                'date1' => '2020-02-02',
+                'date2' => '2020-02-02',
+                'name' => 'done',
+                'report' => null,
+                'ts_invalidated' => '2020-02-02 21:00:00',
+                'status' => 1,
+                'ts_started' => $tsStarted
+            ],
+        ]);
+
+        $t = Fixture::getTracker($idSite, Date::yesterday()->addHour(2)->getDatetime());
+        $t->setUrl('http://someurl.com/abc');
+        Fixture::checkResponse($t->doTrackPageView('some page'));
+
+        $t = Fixture::getTracker($idSite, Date::today()->addHour(2)->getDatetime());
+        $t->setUrl('http://someurl.com/def');
+        Fixture::checkResponse($t->doTrackPageView('some page 2'));
+
+        $mockInvalidateApi = $this->getMockInvalidateApi();
+
+        $archiver = new CronArchive();
+        $archiver->init();
+        $archiver->setApiToInvalidateArchivedReport($mockInvalidateApi);
+
+        $archiver->invalidateRecentDate('yesterday', $idSite);
+        $actualInvalidationCalls = $mockInvalidateApi->getInvalidations();
+
+        $this->assertEquals($expectedInvalidationCalls, $actualInvalidationCalls);
+    }
+
+    public function getInvalidateYesterdayTestData()
+    {
+        $timezones = [
+            'UTC-12',
+            'America/Caracas', // UTC-4
+            'UTC-0.5',
+            'UTC',
+            'Asia/Kathmandu', // UTC+5:45
+            'Australia/Brisbane', // UTC+10
+            'UTC+14',
+        ];
+
+        foreach ($timezones as $timezone) {
+            $offset = Date::getUtcOffset($timezone);
+
+            yield "invalidating yesterday should be skipped if archiving was started after midnight in sites timezone ($timezone)" => [
+                $timezone,
+                Date::factory('2020-02-03 00:12:33')->subSeconds($offset)->getDatetime(),
+                [], // no invalidations
+            ];
+
+            yield "invalidating yesterday should not be skipped if archiving was started before midnight in sites timezone ($timezone)" => [
+                $timezone,
+                Date::factory('2020-02-02 23:48:33')->subSeconds($offset)->getDatetime(),
+                [
+                    [
+                        1,
+                        '2020-02-02',
+                        'day',
+                        false,
+                        false,
+                        false,
+                    ]
+                ],
+            ];
+        }
+    }
+
     /**
      * @dataProvider getArchivingTestData
      */
@@ -1056,9 +1139,10 @@ private function insertInvalidations(array $invalidations)
                 isset($inv['ts_invalidated']) ? $inv['ts_invalidated'] : $now,
                 $inv['report'],
                 isset($inv['status']) ? $inv['status'] : 0,
+                isset($inv['ts_started']) ? $inv['ts_started'] : null,
             ];
-            Db::query("INSERT INTO `$table` (idarchive, name, idsite, date1, date2, period, ts_invalidated, report, status)
-                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", $bind);
+            Db::query("INSERT INTO `$table` (idarchive, name, idsite, date1, date2, period, ts_invalidated, report, status, ts_started)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", $bind);
         }
     }