Merge branch 'master' of github.com:piwik/piwik

core/API/Request.php

@@ -366,7 +366,15 @@ public static function shouldLoadExpanded()
         // we have to load all the child subtables.
         return Common::getRequestVar('filter_column_recursive', false) !== false
             && Common::getRequestVar('filter_pattern_recursive', false) !== false
-            && Common::getRequestVar('flat', false) === false;
+            && !self::shouldLoadFlatten();
+    }
+
+    /**
+     * @return bool
+     */
+    public static function shouldLoadFlatten()
+    {
+        return Common::getRequestVar('flat', false) == 1;
     }
 
     /**

core/Archive.php

@@ -192,9 +192,10 @@ protected function __construct(Parameters $params, $forceIndexedBySite = false,
      *                             or date range (ie, 'YYYY-MM-DD,YYYY-MM-DD').
      * @param bool|false|string $segment Segment definition or false if no segment should be used. {@link Piwik\Segment}
      * @param bool|false|string $_restrictSitesToLogin Used only when running as a scheduled task.
+     * @param bool $skipAggregationOfSubTables Whether the archive, when it is processed, should also aggregate all sub-tables
      * @return Archive
      */
-    public static function build($idSites, $period, $strDate, $segment = false, $_restrictSitesToLogin = false)
+    public static function build($idSites, $period, $strDate, $segment = false, $_restrictSitesToLogin = false, $skipAggregationOfSubTables = false)
     {
         $websiteIds = Site::getIdSitesFromIdSitesString($idSites, $_restrictSitesToLogin);
 
@@ -209,7 +210,7 @@ public static function build($idSites, $period, $strDate, $segment = false, $_re
         $segment = new Segment($segment, $websiteIds);
         $idSiteIsAll = $idSites == self::REQUEST_ALL_WEBSITES_FLAG;
         $isMultipleDate = Period::isMultiplePeriod($strDate, $period);
-        return Archive::factory($segment, $allPeriods, $websiteIds, $idSiteIsAll, $isMultipleDate);
+        return Archive::factory($segment, $allPeriods, $websiteIds, $idSiteIsAll, $isMultipleDate, $skipAggregationOfSubTables);
     }
 
     /**
@@ -231,9 +232,11 @@ public static function build($idSites, $period, $strDate, $segment = false, $_re
      * @param bool $isMultipleDate Whether multiple dates are being queried or not. If true, then
      *                             the result of querying functions will be indexed by period,
      *                             regardless of whether `count($periods) == 1`.
+     * @param bool $skipAggregationOfSubTables Whether the archive should skip aggregation of all sub-tables
+     *
      * @return Archive
      */
-    public static function factory(Segment $segment, array $periods, array $idSites, $idSiteIsAll = false, $isMultipleDate = false)
+    public static function factory(Segment $segment, array $periods, array $idSites, $idSiteIsAll = false, $isMultipleDate = false, $skipAggregationOfSubTables = false)
     {
         $forceIndexedBySite = false;
         $forceIndexedByDate = false;
@@ -244,7 +247,7 @@ public static function factory(Segment $segment, array $periods, array $idSites,
             $forceIndexedByDate = true;
         }
 
-        $params = new Parameters($idSites, $periods, $segment);
+        $params = new Parameters($idSites, $periods, $segment, $skipAggregationOfSubTables);
 
         return new Archive($params, $forceIndexedBySite, $forceIndexedByDate);
     }
@@ -432,16 +435,21 @@ public function getParams()
      * @param string $segment @see {@link build()}
      * @param bool $expanded If true, loads all subtables. See {@link getDataTableExpanded()}
      * @param int|null $idSubtable See {@link getDataTableExpanded()}
+     * @param bool $skipAggregationOfSubTables Whether or not we should skip the aggregation of all sub-tables and only aggregate parent DataTable.
      * @param int|null $depth See {@link getDataTableExpanded()}
      * @return DataTable|DataTable\Map See {@link getDataTable()} and
      *                                 {@link getDataTableExpanded()} for more
      *                                 information
      */
     public static function getDataTableFromArchive($name, $idSite, $period, $date, $segment, $expanded,
-                                                   $idSubtable = null, $depth = null)
+                                                   $idSubtable = null, $skipAggregationOfSubTables = false, $depth = null)
     {
         Piwik::checkUserHasViewAccess($idSite);
-        $archive = Archive::build($idSite, $period, $date, $segment);
+
+        if($skipAggregationOfSubTables && ($expanded || $idSubtable)) {
+            throw new \Exception("Not expected to skipAggregationOfSubTables when expanded=1 or idSubtable is set.");
+        }
+        $archive = Archive::build($idSite, $period, $date, $segment, $_restrictSitesToLogin = false, $skipAggregationOfSubTables);
         if ($idSubtable === false) {
             $idSubtable = null;
         }
@@ -620,7 +628,7 @@ private function cacheArchiveIdsAfterLaunching($archiveGroups, $plugins)
     private function cacheArchiveIdsWithoutLaunching($plugins)
     {
         $idarchivesByReport = ArchiveSelector::getArchiveIds(
-            $this->params->getIdSites(), $this->params->getPeriods(), $this->params->getSegment(), $plugins);
+            $this->params->getIdSites(), $this->params->getPeriods(), $this->params->getSegment(), $plugins, $this->params->isSkipAggregationOfSubTables());
 
         // initialize archive ID cache for each report
         foreach ($plugins as $plugin) {
@@ -644,7 +652,13 @@ private function cacheArchiveIdsWithoutLaunching($plugins)
      */
     private function getDoneStringForPlugin($plugin)
     {
-        return Rules::getDoneStringFlagFor($this->params->getIdSites(), $this->params->getSegment(), $this->getPeriodLabel(), $plugin);
+        return Rules::getDoneStringFlagFor(
+                    $this->params->getIdSites(),
+                    $this->params->getSegment(),
+                    $this->getPeriodLabel(),
+                    $plugin,
+                    $this->params->isSkipAggregationOfSubTables()
+        );
     }
 
     private function getPeriodLabel()
@@ -776,7 +790,7 @@ private static function getPluginForReport($report)
      */
     private function prepareArchive(array $archiveGroups, Site $site, Period $period)
     {
-        $parameters = new ArchiveProcessor\Parameters($site, $period, $this->params->getSegment());
+        $parameters = new ArchiveProcessor\Parameters($site, $period, $this->params->getSegment(), $this->params->isSkipAggregationOfSubTables());
         $archiveLoader = new ArchiveProcessor\Loader($parameters);
 
         $periodString = $period->getRangeString();

core/Archive/Parameters.php

@@ -36,16 +36,22 @@ class Parameters
      */
     private $segment;
 
+    /**
+     * @var bool
+     */
+    private $skipAggregationOfSubTables;
+
     public function getSegment()
     {
         return $this->segment;
     }
 
-    public function __construct($idSites, $periods, Segment $segment)
+    public function __construct($idSites, $periods, Segment $segment, $skipAggregationOfSubTables)
     {
         $this->idSites = $idSites;
         $this->periods = $periods;
         $this->segment = $segment;
+        $this->skipAggregationOfSubTables = $skipAggregationOfSubTables;
     }
 
     public function getPeriods()
@@ -58,5 +64,10 @@ public function getIdSites()
         return $this->idSites;
     }
 
+    public function isSkipAggregationOfSubTables()
+    {
+        return $this->skipAggregationOfSubTables;
+    }
+
 }
 

core/ArchiveProcessor.php

@@ -199,8 +199,14 @@ public function aggregateDataTableRecords($recordNames,
 
             $table = $this->aggregateDataTableRecord($recordName, $columnsAggregationOperation, $columnsToRenameAfterAggregation);
 
-            $nameToCount[$recordName]['level0'] = $table->getRowsCount();
-            $nameToCount[$recordName]['recursive'] = $table->getRowsCountRecursive();
+            $rowsCount = $table->getRowsCount();
+            $nameToCount[$recordName]['level0'] = $rowsCount;
+
+            $rowsCountRecursive = $rowsCount;
+            if($this->isAggregateSubTables()) {
+                $rowsCountRecursive = $table->getRowsCountRecursive();
+            }
+            $nameToCount[$recordName]['recursive'] = $rowsCountRecursive;
 
             $blob = $table->getSerialized($maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable, $columnToSortByBeforeTruncation);
             Common::destroy($table);
@@ -322,7 +328,15 @@ public function insertBlobRecord($name, $values)
      */
     protected function aggregateDataTableRecord($name, $columnsAggregationOperation = null, $columnsToRenameAfterAggregation = null)
     {
-        $dataTable = $this->getArchive()->getDataTableExpanded($name, $idSubTable = null, $depth = null, $addMetadataSubtableId = false);
+        if($this->isAggregateSubTables()) {
+            // By default we shall aggregate all sub-tables.
+            $dataTable = $this->getArchive()->getDataTableExpanded($name, $idSubTable = null, $depth = null, $addMetadataSubtableId = false);
+        } else {
+            // In some cases (eg. Actions plugin when period=range),
+            // for better performance we will only aggregate the parent table
+            $dataTable = $this->getArchive()->getDataTable($name, $idSubTable = null);
+        }
+
         $dataTable = $this->getAggregatedDataTableMap($dataTable, $columnsAggregationOperation);
         $this->renameColumnsAfterAggregation($dataTable, $columnsToRenameAfterAggregation);
         return $dataTable;
@@ -402,7 +416,7 @@ protected function getAggregatedDataTableMap($data, $columnsAggregationOperation
             // as $date => $tableToSum
             $this->aggregatedDataTableMapsAsOne($data, $table);
         } else {
-            $table->addDataTable($data);
+            $table->addDataTable($data, $this->isAggregateSubTables());
         }
         return $table;
     }
@@ -418,7 +432,7 @@ protected function aggregatedDataTableMapsAsOne(Map $map, DataTable $aggregated)
             if($tableToAggregate instanceof Map) {
                 $this->aggregatedDataTableMapsAsOne($tableToAggregate, $aggregated);
             } else {
-                $aggregated->addDataTable($tableToAggregate);
+                $aggregated->addDataTable($tableToAggregate, $this->isAggregateSubTables());
             }
         }
     }
@@ -430,7 +444,7 @@ protected function renameColumnsAfterAggregation(DataTable $table, $columnsToRen
             $columnsToRenameAfterAggregation = self::$columnsToRenameAfterAggregation;
         }
         foreach ($columnsToRenameAfterAggregation as $oldName => $newName) {
-            $table->renameColumn($oldName, $newName);
+            $table->renameColumn($oldName, $newName, $this->isAggregateSubTables());
         }
     }
 
@@ -464,4 +478,12 @@ protected function getAggregatedNumericMetrics($columns, $operationToApply)
         }
         return $metrics;
     }
+
+    /**
+     * @return bool
+     */
+    protected function isAggregateSubTables()
+    {
+        return !$this->getParams()->isSkipAggregationOfSubTables();
+    }
 }

core/ArchiveProcessor/Loader.php

@@ -163,12 +163,8 @@ protected function loadExistingArchiveIdFromDb()
         if ($this->isArchivingForcedToTrigger()) {
             return $noArchiveFound;
         }
-        $site = $this->params->getSite();
-        $period = $this->params->getPeriod();
-        $segment = $this->params->getSegment();
-        $requestedPlugin = $this->params->getRequestedPlugin();
 
-        $idAndVisits = ArchiveSelector::getArchiveIdAndVisits($site, $period, $segment, $minDatetimeArchiveProcessedUTC, $requestedPlugin);
+        $idAndVisits = ArchiveSelector::getArchiveIdAndVisits($this->params, $minDatetimeArchiveProcessedUTC);
         if (!$idAndVisits) {
             return $noArchiveFound;
         }

core/ArchiveProcessor/Parameters.php

@@ -48,11 +48,12 @@ class Parameters
      * 
      * @ignore
      */
-    public function __construct(Site $site, Period $period, Segment $segment)
+    public function __construct(Site $site, Period $period, Segment $segment, $skipAggregationOfSubTables = false)
     {
         $this->site = $site;
         $this->period = $period;
         $this->segment = $segment;
+        $this->skipAggregationOfSubTables = $skipAggregationOfSubTables;
     }
 
     /**
@@ -168,6 +169,11 @@ public function isSingleSite()
         return count($this->getIdSites()) == 1;
     }
 
+    public function isSkipAggregationOfSubTables()
+    {
+        return $this->skipAggregationOfSubTables;
+    }
+
     public function logStatusDebug($isTemporary)
     {
         $temporary = 'definitive archive';

core/ArchiveProcessor/Rules.php

@@ -50,10 +50,10 @@ class Rules
      * @param string $plugin
      * @return string
      */
-    public static function getDoneStringFlagFor(array $idSites, $segment, $periodLabel, $plugin)
+    public static function getDoneStringFlagFor(array $idSites, $segment, $periodLabel, $plugin, $isSkipAggregationOfSubTables)
     {
         if (!self::shouldProcessReportsAllPlugins($idSites, $segment, $periodLabel)) {
-            return self::getDoneFlagArchiveContainsOnePlugin($segment, $plugin);
+            return self::getDoneFlagArchiveContainsOnePlugin($segment, $plugin, $isSkipAggregationOfSubTables);
         }
         return self::getDoneFlagArchiveContainsAllPlugins($segment);
     }
@@ -98,28 +98,47 @@ private static function getSegmentsToProcess($idSites)
         return $segmentsToProcess;
     }
 
-    private static function getDoneFlagArchiveContainsOnePlugin(Segment $segment, $plugin)
+    private static function getDoneFlagArchiveContainsOnePlugin(Segment $segment, $plugin, $isSkipAggregationOfSubTables = false)
     {
-        return 'done' . $segment->getHash() . '.' . $plugin;
+        $partial = self::isFlagArchivePartial($plugin, $isSkipAggregationOfSubTables);
+        return 'done' . $segment->getHash() . '.' . $plugin . $partial ;
     }
 
     private static function getDoneFlagArchiveContainsAllPlugins(Segment $segment)
     {
         return 'done' . $segment->getHash();
     }
 
+    /**
+     * @param $plugin
+     * @param $isSkipAggregationOfSubTables
+     * @return string
+     */
+    private static function isFlagArchivePartial($plugin, $isSkipAggregationOfSubTables)
+    {
+        $partialArchive = '';
+        if ($plugin != "VisitsSummary" // VisitsSummary is always called when segmenting and should not have its own .partial archive
+            && $isSkipAggregationOfSubTables
+        ) {
+            $partialArchive = '.partial';
+        }
+        return $partialArchive;
+    }
+
     /**
      * @param array $plugins
      * @param $segment
      * @return array
      */
-    public static function getDoneFlags(array $plugins, $segment)
+    public static function getDoneFlags(array $plugins, Segment $segment, $isSkipAggregationOfSubTables)
     {
         $doneFlags = array();
         $doneAllPlugins = self::getDoneFlagArchiveContainsAllPlugins($segment);
         $doneFlags[$doneAllPlugins] = $doneAllPlugins;
+
+        $plugins = array_unique($plugins);
         foreach ($plugins as $plugin) {
-            $doneOnePlugin = self::getDoneFlagArchiveContainsOnePlugin($segment, $plugin);
+            $doneOnePlugin = self::getDoneFlagArchiveContainsOnePlugin($segment, $plugin, $isSkipAggregationOfSubTables);
             $doneFlags[$plugin] = $doneOnePlugin;
         }
         return $doneFlags;