When importing visits on a day different from the visits day, invalid…

…ate the archived reports

core/CronArchive.php

@@ -12,10 +12,12 @@
 use Piwik\ArchiveProcessor\Rules;
 use Piwik\CronArchive\FixedSiteIds;
 use Piwik\CronArchive\SharedSiteIds;
+use Piwik\DataAccess\ArchiveInvalidator;
 use Piwik\Exception\UnexpectedWebsiteFoundException;
 use Piwik\Metrics\Formatter;
 use Piwik\Period\Factory as PeriodFactory;
 use Piwik\DataAccess\InvalidatedReports;
+use Piwik\Plugins\CoreAdminHome\API as CoreAdminHomeAPI;
 use Piwik\Plugins\SitesManager\API as APISitesManager;
 use Piwik\Plugins\UsersManager\API as APIUsersManager;
 use Piwik\Plugins\UsersManager\UserPreferences;
@@ -85,6 +87,8 @@ class CronArchive
     private $lastSuccessRunTimestamp = false;
     private $errors = array();
 
+    private $apiToInvalidateArchivedReport;
+
     const NO_ERROR = "no error";
 
     public $testmode = false;
@@ -256,6 +260,8 @@ public function init()
             $this->log("- Will only process the following periods: " . implode(", ", $this->shouldArchiveOnlySpecificPeriods) . " (--force-periods)");
         }
 
+        $this->invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain();
+
         $websitesIds = $this->initWebsiteIds();
         $this->filterWebsiteIds($websitesIds);
 
@@ -951,6 +957,40 @@ public function filterWebsiteIds(&$websiteIds)
         Piwik::postEvent('CronArchive.filterWebsiteIds', array(&$websiteIds));
     }
 
+    /**
+     * @internal
+     */
+    public function setApiToInvalidateArchivedReport($api)
+    {
+        $this->apiToInvalidateArchivedReport = $api;
+    }
+
+    private function getApiToInvalidateArchivedReport()
+    {
+        if ($this->apiToInvalidateArchivedReport) {
+            return $this->apiToInvalidateArchivedReport;
+        }
+
+        return CoreAdminHomeAPI::getInstance();
+    }
+
+    public function invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain()
+    {
+        $invalidator  = new ArchiveInvalidator();
+        $sitesPerDays = $invalidator->getRememberedArchivedReportsThatShouldBeInvalidated();
+
+        foreach ($sitesPerDays as $date => $siteIds) {
+            $listSiteIds = implode(',', $siteIds);
+
+            try {
+                $this->log('Will invalidate archived reports for ' . $date . ' for following siteIds: ' . $listSiteIds);
+                $this->getApiToInvalidateArchivedReport()->invalidateArchivedReports($siteIds, $date);
+            } catch (Exception $e) {
+                $this->log('Failed to invalidate archived reports: ' . $e->getMessage());
+            }
+        }
+    }
+
     /**
      *  Returns the list of sites to loop over and archive.
      *  @return array
@@ -998,7 +1038,7 @@ public function isTokenAuthSuperUserToken($token_auth)
         return in_array($token_auth, $this->validTokenAuths);
     }
 
-    private function initPiwikHost($piwikUrl = false)
+    protected function initPiwikHost($piwikUrl = false)
     {
         // If core:archive command run as a web cron, we use the current hostname+path
         if (empty($piwikUrl)) {
@@ -1150,7 +1190,7 @@ private function addWebsiteIdsInTimezoneWithNewDay($websiteIds)
     /**
      *  Test that the specified piwik URL is a valid Piwik endpoint.
      */
-    private function checkPiwikUrlIsValid()
+    protected function checkPiwikUrlIsValid()
     {
         $response = $this->request("?module=API&method=API.getDefaultMetricTranslations&format=original&serialize=1");
         $responseUnserialized = @unserialize($response);

core/DataAccess/ArchiveInvalidator.php

@@ -9,9 +9,9 @@
 
 namespace Piwik\DataAccess;
 
-
 use Piwik\Date;
 use Piwik\Db;
+use Piwik\Option;
 use Piwik\Plugins\PrivacyManager\PrivacyManager;
 use Piwik\Period;
 use Piwik\Period\Week;
@@ -32,6 +32,73 @@ class ArchiveInvalidator {
     private $minimumDateWithLogs = false;
     private $invalidDates = array();
 
+    private $rememberArchivedReportIdStart = 'report_to_invalidate_';
+
+    public function rememberToInvalidateArchivedReportsLater($idSite, Date $date)
+    {
+        $key   = $this->buildRememberArchivedReportId($idSite, $date->toString());
+        $value = Option::get($key);
+
+        // we do not really have to get the value first. we could simply always try to call set() and it would update or
+        // insert the record if needed but we do not want to lock the table (especially since there are still some
+        // MyISAM installations)
+
+        if (false === $value) {
+            Option::set($key, '1');
+        }
+    }
+
+    public function getRememberedArchivedReportsThatShouldBeInvalidated()
+    {
+        $reports = Option::getLike($this->rememberArchivedReportIdStart . '%_%');
+
+        $sitesPerDay = array();
+
+        foreach ($reports as $report => $value) {
+            $report = str_replace($this->rememberArchivedReportIdStart, '', $report);
+            $report = explode('_', $report);
+            $siteId = (int) $report[0];
+            $date   = $report[1];
+
+            if (empty($sitesPerDay[$date])) {
+                $sitesPerDay[$date] = array();
+            }
+
+            $sitesPerDay[$date][] = $siteId;
+        }
+
+        return $sitesPerDay;
+    }
+
+    private function buildRememberArchivedReportId($idSite, $date)
+    {
+        $id  = $this->buildRememberArchivedReportIdForSite($idSite);
+        $id .= '_' . trim($date);
+
+        return $id;
+    }
+
+    private function buildRememberArchivedReportIdForSite($idSite)
+    {
+        return $this->rememberArchivedReportIdStart . (int) $idSite;
+    }
+
+    public function forgetRememberedArchivedReportsToInvalidateForSite($idSite)
+    {
+        $id = $this->buildRememberArchivedReportIdForSite($idSite) . '_%';
+        Option::deleteLike($id);
+    }
+
+    /**
+     * @internal
+     */
+    public function forgetRememberedArchivedReportsToInvalidate($idSite, Date $date)
+    {
+        $id = $this->buildRememberArchivedReportId($idSite, $date->toString());
+
+        Option::delete($id);
+    }
+
     /**
      * @param $idSites array
      * @param $dates string
@@ -52,6 +119,12 @@ public function markArchivesAsInvalidated(array $idSites, $dates, $period)
 
         $this->persistInvalidatedArchives($idSites, $datesByMonth);
 
+        foreach ($idSites as $idSite) {
+            foreach ($datesToInvalidate as $date) {
+                $this->forgetRememberedArchivedReportsToInvalidate($idSite, $date);
+            }
+        }
+
         return $this->makeOutputLogs();
     }
 
@@ -105,7 +178,7 @@ private function markArchivesInvalidatedFor($idSites, $period, $datesByMonth)
     /**
      * Ensure the specified dates are valid.
      * Store invalid date so we can log them
-     * @param $dates string
+     * @param array $dates
      * @return Date[]
      */
     private function getDatesToInvalidateFromString($dates)
@@ -129,6 +202,7 @@ private function getDatesToInvalidateFromString($dates)
                 $this->invalidDates[] = $theDate;
             }
         }
+
         return $toInvalidate;
     }
 

core/Site.php

@@ -133,7 +133,12 @@ protected static function setSite($idSite, $infoSite)
     public static function setSitesFromArray($sites)
     {
         foreach ($sites as $site) {
-            self::setSite($site['idsite'], $site);
+            $idSite = null;
+            if (!empty($site['idsite'])) {
+                $idSite = $site['idsite'];
+            }
+
+            self::setSite($idSite, $site);
         }
     }
 

core/Tracker/Cache.php

@@ -60,7 +60,7 @@ static function getCacheWebsiteAttributes($idSite)
             return array();
         }
 
-        $idSite = (int)$idSite;
+        $idSite = (int) $idSite;
         if ($idSite <= 0) {
             return array();
         }

core/Tracker/Visit.php

@@ -11,6 +11,9 @@
 
 use Piwik\Common;
 use Piwik\Config;
+use Piwik\DataAccess\ArchiveInvalidator;
+use Piwik\Date;
+use Piwik\Exception\UnexpectedWebsiteFoundException;
 use Piwik\Network\IPUtils;
 use Piwik\Piwik;
 use Piwik\Plugin\Dimension\VisitDimension;
@@ -218,6 +221,8 @@ public function handle()
         }
         unset($this->goalManager);
         unset($action);
+
+        $this->markArchivedReportsAsInvalidIfArchiveAlreadyFinished();
     }
 
     /**
@@ -626,4 +631,23 @@ public function isVisitNew(Visitor $visitor, Action $action = null)
 
         return !$visitor->isVisitorKnown();
     }
+
+    private function markArchivedReportsAsInvalidIfArchiveAlreadyFinished()
+    {
+        $idSite = (int) $this->request->getIdSite();
+        $time   = $this->request->getCurrentTimestamp();
+
+        try {
+            $site = Cache::getCacheWebsiteAttributes($idSite);
+        } catch (UnexpectedWebsiteFoundException $e) {
+            return;
+        }
+
+        $date = Date::factory((int) $time, $site['timezone']);
+
+        if (!$date->isToday()) { // we don't have to handle in case date is in future as it is not allowed by tracker
+            $invalidReport = new ArchiveInvalidator();
+            $invalidReport->rememberToInvalidateArchivedReportsLater($idSite, $date);
+        }
+    }
 }

misc/log-analytics/import_logs.py

@@ -1443,26 +1443,6 @@ def _on_tracking_failure(self, response, data):
 
         return response['message']
 
-    @staticmethod
-    def invalidate_reports():
-        if config.options.dry_run or not stats.dates_recorded:
-            return
-
-        if config.options.invalidate_dates is not None:
-            dates = [date for date in config.options.invalidate_dates.split(',') if date]
-        else:
-            dates = [date.strftime('%Y-%m-%d') for date in stats.dates_recorded]
-        if dates:
-            print '\nPurging Piwik archives for dates: ' + ' '.join(dates)
-            result = piwik.call_api(
-                'CoreAdminHome.invalidateArchivedReports',
-                dates=','.join(dates),
-                idSites=','.join(str(site_id) for site_id in stats.piwik_sites),
-            )
-            print('\nTo re-process these reports with your newly imported data, execute the following command: \n'
-                  '$ /path/to/piwik/console core:archive --url=http://example/piwik --force-all-websites --force-all-periods=315576000 --force-date-last-n=1000'
-                  '\nReference: http://piwik.org/docs/setup-auto-archiving/ ')
-
 class Hit(object):
     """
     It's a simple container.
@@ -1899,10 +1879,6 @@ def main():
     if config.options.show_progress:
         stats.stop_monitor()
 
-    try:
-        Recorder.invalidate_reports()
-    except Piwik.Error, e:
-        pass
     stats.print_summary()
 
 def fatal_error(error, filename=None, lineno=None):

plugins/CoreAdminHome/tests/Framework/Mock/API.php

@@ -0,0 +1,26 @@
+<?php
+/**
+* Piwik - free/libre analytics platform
+*
+* @link http://piwik.org
+* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+*/
+
+namespace Piwik\Plugins\CoreAdminHome\tests\Framework\Mock;
+
+use Piwik\Tracker;
+
+class API extends \Piwik\Plugins\CoreAdminHome\API
+{
+    private $invalidatedReports = array();
+
+    public function invalidateArchivedReports($idSites, $dates, $period = false)
+    {
+        $this->invalidatedReports[] = func_get_args();
+    }
+
+    public function getInvalidatedReports()
+    {
+        return $this->invalidatedReports;
+    }
+}

plugins/SitesManager/API.php

@@ -182,6 +182,7 @@ public function getSiteFromId($idSite)
         $site = $this->getModel()->getSiteFromId($idSite);
 
         Site::setSitesFromArray(array($site));
+
         return $site;
     }
 
@@ -599,9 +600,6 @@ public function deleteSite($idSite)
 
         $this->getModel()->deleteSite($idSite);
 
-        // we do not delete logs here on purpose (you can run these queries on the log_ tables to delete all data)
-        Cache::deleteCacheWebsiteAttributes($idSite);
-
         /**
          * Triggered after a site has been deleted.
          *