Merge pull request #153 from piwik/ratio_1816

Add a new column in all reports for % percentage of visits

core/API/DataTableManipulator.php

@@ -128,26 +128,8 @@ protected function loadSubtable($dataTable, $row)
             }
         }
 
-        $class = Request::getClassNameAPI($this->apiModule);
         $method = $this->getApiMethodForSubtable();
-
-        $this->manipulateSubtableRequest($request);
-        $request['serialize'] = 0;
-        $request['expanded'] = 0;
-
-        // don't want to run recursive filters on the subtables as they are loaded,
-        // otherwise the result will be empty in places (or everywhere). instead we
-        // run it on the flattened table.
-        unset($request['filter_pattern_recursive']);
-
-        $dataTable = Proxy::getInstance()->call($class, $method, $request);
-        $response = new ResponseBuilder($format = 'original', $request);
-        $dataTable = $response->getResponse($dataTable);
-        if (method_exists($dataTable, 'applyQueuedFilters')) {
-            $dataTable->applyQueuedFilters();
-        }
-
-        return $dataTable;
+        return $this->callApiAndReturnDataTable($this->apiModule, $method, $request);
     }
 
     /**
@@ -158,7 +140,7 @@ protected function loadSubtable($dataTable, $row)
      * @param $request
      * @return
      */
-    protected abstract function manipulateSubtableRequest(&$request);
+    protected abstract function manipulateSubtableRequest($request);
 
     /**
      * Extract the API method for loading subtables from the meta data
@@ -177,4 +159,27 @@ private function getApiMethodForSubtable()
         }
         return $this->apiMethodForSubtable;
     }
+
+    protected function callApiAndReturnDataTable($apiModule, $method, $request)
+    {
+        $class = Request::getClassNameAPI($apiModule);
+
+        $request = $this->manipulateSubtableRequest($request);
+        $request['serialize'] = 0;
+        $request['expanded'] = 0;
+
+        // don't want to run recursive filters on the subtables as they are loaded,
+        // otherwise the result will be empty in places (or everywhere). instead we
+        // run it on the flattened table.
+        unset($request['filter_pattern_recursive']);
+
+        $dataTable = Proxy::getInstance()->call($class, $method, $request);
+        $response = new ResponseBuilder($format = 'original', $request);
+        $dataTable = $response->getResponse($dataTable);
+        if (method_exists($dataTable, 'applyQueuedFilters')) {
+            $dataTable->applyQueuedFilters();
+        }
+
+        return $dataTable;
+    }
 }

core/API/DataTableManipulator/Flattener.php

@@ -128,8 +128,10 @@ private function flattenRow(Row $row, DataTable $dataTable,
      *
      * @param array $request
      */
-    protected function manipulateSubtableRequest(&$request)
+    protected function manipulateSubtableRequest($request)
     {
         unset($request['flat']);
+
+        return $request;
     }
 }

core/API/DataTableManipulator/LabelFilter.php

@@ -103,9 +103,11 @@ private function doFilterRecursiveDescend($labelParts, $dataTable)
      *
      * @param $request
      */
-    protected function manipulateSubtableRequest(&$request)
+    protected function manipulateSubtableRequest($request)
     {
         unset($request['label']);
+
+        return $request;
     }
 
     /**

core/API/DataTableManipulator/ReportTotalsCalculator.php

@@ -0,0 +1,245 @@
+<?php
+/**
+ * Piwik - Open source web analytics
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ * @category Piwik
+ * @package Piwik
+ */
+namespace Piwik\API\DataTableManipulator;
+
+use Piwik\API\DataTableManipulator;
+use Piwik\DataTable;
+use Piwik\DataTable\Row;
+use Piwik\DataTable\Filter;
+use Piwik\Period\Range;
+use Piwik\Period;
+use Piwik\Piwik;
+use Piwik\Metrics;
+use Piwik\Plugins\API\API;
+
+/**
+ * This class is responsible for setting the metadata property 'totals' on each dataTable if the report
+ * has a dimension. 'Totals' means it tries to calculate the total report value for each metric. For each
+ * the total number of visits, actions, ... for a given report / dataTable.
+ *
+ * @package Piwik
+ * @subpackage Piwik_API
+ */
+class ReportTotalsCalculator extends DataTableManipulator
+{
+    /**
+     * Cached report metadata array.
+     * @var array
+     */
+    private static $reportMetadata = array();
+
+    /**
+     * @param  DataTable $table
+     * @return \Piwik\DataTable|\Piwik\DataTable\Map
+     */
+    public function calculate($table)
+    {
+        // apiModule and/or apiMethod is empty for instance in case when flat=1 is called. Basically whenever a
+        // datamanipulator calls the API and wants the dataTable in return, see callApiAndReturnDataTable().
+        // it is also not set for some settings API request etc.
+        if (empty($this->apiModule) || empty($this->apiMethod)) {
+            return $table;
+        }
+
+        return $this->manipulate($table);
+    }
+
+    /**
+     * Adds ratio metrics if possible.
+     *
+     * @param  DataTable $dataTable
+     * @return DataTable
+     */
+    protected function manipulateDataTable($dataTable)
+    {
+        $report = $this->findCurrentReport();
+
+        if (!empty($report) && empty($report['dimension'])) {
+            // we currently do not calculate the total value for reports having no dimension
+            return $dataTable;
+        }
+
+        // Array [readableMetric] => [summed value]
+        $totalValues = array();
+
+        $firstLevelTable    = $this->makeSureToWorkOnFirstLevelDataTable($dataTable);
+        $metricsToCalculate = Metrics::getMetricIdsToProcessReportTotal();
+
+        foreach ($metricsToCalculate as $metricId) {
+            if (!$this->hasDataTableMetric($firstLevelTable, $metricId)) {
+                continue;
+            }
+
+            foreach ($firstLevelTable->getRows() as $row) {
+                $totalValues = $this->sumColumnValueToTotal($row, $metricId, $totalValues);
+            }
+        }
+
+        $dataTable->setMetadata('totals', $totalValues);
+
+        return $dataTable;
+    }
+
+    private function hasDataTableMetric(DataTable $dataTable, $metricId)
+    {
+        $firstRow = $dataTable->getFirstRow();
+
+        if (empty($firstRow)) {
+            return false;
+        }
+
+        if (false === $this->getColumn($firstRow, $metricId)) {
+            return false;
+        }
+
+        return true;
+    }
+
+    /**
+     * Returns column from a given row.
+     * Will work with 2 types of datatable
+     * - raw datatables coming from the archive DB, which columns are int indexed
+     * - datatables processed resulting of API calls, which columns have human readable english names
+     *
+     * @param Row|array $row
+     * @param int $columnIdRaw see consts in Metrics::
+     * @return mixed  Value of column, false if not found
+     */
+    private function getColumn($row, $columnIdRaw)
+    {
+        $columnIdReadable = Metrics::getReadableColumnName($columnIdRaw);
+
+        if ($row instanceof Row) {
+            $raw = $row->getColumn($columnIdRaw);
+            if ($raw !== false) {
+                return $raw;
+            }
+
+            return $row->getColumn($columnIdReadable);
+        }
+
+        return false;
+    }
+
+    private function makeSureToWorkOnFirstLevelDataTable($table)
+    {
+        if (!array_key_exists('idSubtable', $this->request)) {
+            return $table;
+        }
+
+        $firstLevelReport = $this->findFirstLevelReport();
+
+        if (empty($firstLevelReport)) {
+            // it is not a subtable report
+            $module = $this->apiModule;
+            $action = $this->apiMethod;
+        } else {
+            $module = $firstLevelReport['module'];
+            $action = $firstLevelReport['action'];
+        }
+
+        $request = $this->request;
+
+        /** @var \Piwik\Period $period */
+        $period = $table->getMetadata('period');
+
+        if (!empty($period)) {
+            // we want a dataTable, not a dataTable\map
+            if (Period::isMultiplePeriod($request['date'], $request['period']) || 'range' == $period->getLabel()) {
+                $request['date']   = $period->getRangeString();
+                $request['period'] = 'range';
+            } else {
+                $request['date']   = $period->getDateStart()->toString();
+                $request['period'] = $period->getLabel();
+            }
+        }
+
+        return $this->callApiAndReturnDataTable($module, $action, $request);
+    }
+
+    private function sumColumnValueToTotal(Row $row, $metricId, $totalValues)
+    {
+        $value = $this->getColumn($row, $metricId);
+
+        if (false === $value) {
+
+            return $totalValues;
+        }
+
+        $metricName = Metrics::getReadableColumnName($metricId);
+
+        if (array_key_exists($metricName, $totalValues)) {
+            $totalValues[$metricName] += $value;
+        } else {
+            $totalValues[$metricName] = $value;
+        }
+
+        return $totalValues;
+    }
+
+    /**
+     * Make sure to get all rows of the first level table.
+     *
+     * @param array $request
+     */
+    protected function manipulateSubtableRequest($request)
+    {
+        $request['totals']        = 0;
+        $request['expanded']      = 0;
+        $request['filter_limit']  = -1;
+        $request['filter_offset'] = 0;
+
+        $parametersToRemove = array('flat', 'idSubtable');
+
+        foreach ($parametersToRemove as $param) {
+            if (array_key_exists($param, $request)) {
+                unset($request[$param]);
+            }
+        }
+
+        return $request;
+    }
+
+    private function getReportMetadata()
+    {
+        if (!empty(static::$reportMetadata)) {
+            return static::$reportMetadata;
+        }
+
+        static::$reportMetadata = API::getInstance()->getReportMetadata();
+
+        return static::$reportMetadata;
+    }
+
+    private function findCurrentReport()
+    {
+        foreach ($this->getReportMetadata() as $report) {
+            if ($this->apiMethod == $report['action']
+                && $this->apiModule == $report['module']) {
+
+                return $report;
+            }
+        }
+    }
+
+    private function findFirstLevelReport()
+    {
+        foreach ($this->getReportMetadata() as $report) {
+            if (!empty($report['actionToLoadSubTables'])
+                && $this->apiMethod == $report['actionToLoadSubTables']
+                && $this->apiModule == $report['module']
+            ) {
+
+                return $report;
+            }
+        }
+    }
+}

core/API/ResponseBuilder.php

@@ -13,6 +13,7 @@
 use Exception;
 use Piwik\API\DataTableManipulator\Flattener;
 use Piwik\API\DataTableManipulator\LabelFilter;
+use Piwik\API\DataTableManipulator\ReportTotalsCalculator;
 use Piwik\Common;
 use Piwik\DataTable\Renderer\Json;
 use Piwik\DataTable\Renderer;
@@ -299,6 +300,11 @@ protected function handleDataTable($datatable)
             $datatable = $flattener->flatten($datatable);
         }
 
+        if (1 == Common::getRequestVar('totals', '1', 'integer', $this->request)) {
+            $genericFilter = new ReportTotalsCalculator($this->apiModule, $this->apiMethod, $this->request);
+            $datatable     = $genericFilter->calculate($datatable);
+        }
+
         // if the flag disable_generic_filters is defined we skip the generic filters
         if (0 == Common::getRequestVar('disable_generic_filters', '0', 'string', $this->request)) {
             $genericFilter = new DataTableGenericFilter($this->request);

core/Metrics.php

@@ -286,6 +286,36 @@ static public function getDefaultProcessedMetrics()
         return array_map(array('\\Piwik\\Piwik','translate'), $translations);
     }
 
+    static public function getReadableColumnName($columnIdRaw)
+    {
+        $mappingIdToName = self::$mappingFromIdToName;
+
+        if (array_key_exists($columnIdRaw, $mappingIdToName)) {
+
+            return $mappingIdToName[$columnIdRaw];
+        }
+
+        return $columnIdRaw;
+    }
+
+    static public function getMetricIdsToProcessReportTotal()
+    {
+        return array(
+            self::INDEX_NB_VISITS,
+            self::INDEX_NB_UNIQ_VISITORS,
+            self::INDEX_NB_ACTIONS,
+            self::INDEX_PAGE_NB_HITS,
+            self::INDEX_NB_VISITS_CONVERTED,
+            self::INDEX_NB_CONVERSIONS,
+            self::INDEX_BOUNCE_COUNT,
+            self::INDEX_PAGE_ENTRY_BOUNCE_COUNT,
+            self::INDEX_PAGE_ENTRY_NB_VISITS,
+            self::INDEX_PAGE_ENTRY_NB_ACTIONS,
+            self::INDEX_PAGE_EXIT_NB_VISITS,
+            self::INDEX_PAGE_EXIT_NB_UNIQ_VISITORS
+        );
+    }
+
     static public function getDefaultMetricsDocumentation()
     {
         $documentation = array(