Skip to content

Commit

Permalink
Refs #4278 Rewriting aggregateNumericMetrics() to use DataTable aggre…
Browse files Browse the repository at this point in the history 
…gation only (code reuse)
  • Loading branch information
@mattab
mattab committed Nov 13, 2013
1 parent 9caddfb commit f197294
Show file tree
Hide file tree
Showing 11 changed files with 225 additions and 171 deletions.
2 changes: 1 addition & 1 deletion core/API/DataTableManipulator.php
View file
Expand Up @@ -55,7 +55,7 @@ public function __construct($apiModule = false, $apiMethod = false, $request = a

/**
* This method can be used by subclasses to iterate over data tables that might be
* data table arrays. It calls back the template method self::doManipulate for each table.
* data table maps. It calls back the template method self::doManipulate for each table.
* This way, data table arrays can be handled in a transparent fashion.
*
* @param DataTable\Map|DataTable $dataTable
Expand Down
2 changes: 1 addition & 1 deletion core/Archive.php
View file
Expand Up @@ -392,7 +392,7 @@ public function getDataTable($name, $idSubtable = null)
* loaded. Those subtables, however, will NOT have their subtables loaded.
* @param bool $addMetadataSubtableId Whether to add the database subtable ID as metadata to each datatable,
* or not.
* @return DataTable
* @return DataTable|DataTable\Map
*/
public function getDataTableExpanded($name, $idSubtable = null, $depth = null, $addMetadataSubtableId = true)
{
Expand Down
16 changes: 2 additions & 14 deletions core/Archive/Parameters.php
View file
Expand Up @@ -45,8 +45,8 @@ public function getSegment()

public function __construct($idSites, $periods, Segment $segment)
{
$this->idSites = $this->getAsNonEmptyArray($idSites, 'idSites');
$this->periods = $this->getAsNonEmptyArray($periods, 'periods');
$this->idSites = $idSites;
$this->periods = $periods;
$this->segment = $segment;
}

Expand All @@ -60,17 +60,5 @@ public function getIdSites()
return $this->idSites;
}

private function getAsNonEmptyArray($array, $paramName)
{
if (!is_array($array)) {
$array = array($array);
}

if (empty($array)) {
throw new Exception("Archive::__construct: \$$paramName is empty.");
}

return $array;
}
}

206 changes: 99 additions & 107 deletions core/ArchiveProcessor.php
View file
Expand Up @@ -16,6 +16,7 @@
use Piwik\DataAccess\ArchiveWriter;
use Piwik\DataAccess\LogAggregator;
use Piwik\DataTable\Manager;
use Piwik\DataTable\Map;
use Piwik\Db;
use Piwik\Period;

Expand Down Expand Up @@ -133,14 +134,13 @@ public function __construct(Parameters $params, ArchiveWriter $archiveWriter)
protected function getArchive()
{
if(empty($this->archive)) {
$subPeriods = $this->params->getPeriod()->getSubperiods();
$idSite = $this->params->getSite()->getId();
$this->archive = Archive::factory($this->params->getSegment(), $subPeriods, array($idSite));
$subPeriods = $this->params->getSubPeriods();
$idSites = $this->params->getIdSites();
$this->archive = Archive::factory($this->params->getSegment(), $subPeriods, $idSites);
}
return $this->archive;
}


public function setNumberOfVisits($visits, $visitsConverted)
{
$this->numberOfVisits = $visits;
Expand Down Expand Up @@ -175,7 +175,7 @@ public function getLogAggregator()
* These columns will be renamed as per this mapping.
* @var array
*/
protected static $invalidSummedColumnNameToRenamedName = array(
protected static $columnsToRenameAfterAggregation = array(
Metrics::INDEX_NB_UNIQ_VISITORS => Metrics::INDEX_SUM_DAILY_NB_UNIQ_VISITORS
);

Expand All @@ -189,8 +189,8 @@ public function getLogAggregator()
* @param int $maximumRowsInDataTableLevelZero Maximum number of rows allowed in the top level DataTable.
* @param int $maximumRowsInSubDataTable Maximum number of rows allowed in each subtable.
* @param string $columnToSortByBeforeTruncation The name of the column to sort by before truncating a DataTable.
* @param array $columnAggregationOperations Operations for aggregating columns, @see Row::sumRow().
* @param array $invalidSummedColumnNameToRenamedName For columns that must change names when summed because they
* @param array $columnsAggregationOperation Operations for aggregating columns, @see Row::sumRow().
* @param array $columnsToRenameAfterAggregation For columns that must change names when summed because they
* cannot be summed, eg,
* `array('nb_uniq_visitors' => 'sum_daily_nb_uniq_visitors')`.
* @return array Returns the row counts of each aggregated report before truncation, eg,
Expand All @@ -209,8 +209,8 @@ public function aggregateDataTableRecords($recordNames,
$maximumRowsInDataTableLevelZero = null,
$maximumRowsInSubDataTable = null,
$columnToSortByBeforeTruncation = null,
&$columnAggregationOperations = null,
$invalidSummedColumnNameToRenamedName = null)
&$columnsAggregationOperation = null,
$columnsToRenameAfterAggregation = null)
{
// We clean up below all tables created during this function call (and recursive calls)
$latestUsedTableId = Manager::getInstance()->getMostRecentTableId();
Expand All @@ -219,7 +219,7 @@ public function aggregateDataTableRecords($recordNames,
}
$nameToCount = array();
foreach ($recordNames as $recordName) {
$table = $this->aggregateDataTableRecord($recordName, $invalidSummedColumnNameToRenamedName, $columnAggregationOperations);
$table = $this->aggregateDataTableRecord($recordName, $columnsAggregationOperation, $columnsToRenameAfterAggregation);

$nameToCount[$recordName]['level0'] = $table->getRowsCount();
$nameToCount[$recordName]['recursive'] = $table->getRowsCountRecursive();
Expand Down Expand Up @@ -254,26 +254,18 @@ public function aggregateDataTableRecords($recordNames,
*/
public function aggregateNumericMetrics($columns, $operationToApply = false)
{
if (!is_array($columns)) {
$columns = array($columns);
}
$data = $this->getArchive()->getNumeric($columns);
$operationForColumn = $this->getOperationForColumns($columns, $operationToApply);
$results = $this->aggregateDataArray($data, $operationForColumn);
$results = $this->defaultColumnsToZero($columns, $results);
$this->enrichWithUniqueVisitorsMetric($results);
$metrics = $this->getAggregatedNumericMetrics($columns, $operationToApply);

foreach ($results as $name => $value) {
$this->archiveWriter->insertRecord($name, $value);
foreach($metrics as $column => $value) {
$this->archiveWriter->insertRecord($column, $value);
}

// if asked for only one field to sum
if (count($results) == 1) {
return reset($results);
if (count($metrics) == 1) {
return reset($metrics);
}

// returns the array of records once summed
return $results;
return $metrics;
}

public function getNumberOfVisits()
Expand Down Expand Up @@ -345,33 +337,14 @@ public function insertBlobRecord($name, $values)
* All these DataTables are then added together, and the resulting DataTable is returned.
*
* @param string $name
* @param array $invalidSummedColumnNameToRenamedName columns in the array (old name, new name) to be renamed as the sum operation is not valid on them (eg. nb_uniq_visitors->sum_daily_nb_uniq_visitors)
* @param array $columnAggregationOperations Operations for aggregating columns, @see Row::sumRow()
* @param array $columnsAggregationOperation Operations for aggregating columns, @see Row::sumRow()
* @param array $columnsToRenameAfterAggregation columns in the array (old name, new name) to be renamed as the sum operation is not valid on them (eg. nb_uniq_visitors->sum_daily_nb_uniq_visitors)
* @return DataTable
*/
protected function aggregateDataTableRecord($name, $invalidSummedColumnNameToRenamedName, $columnAggregationOperations = null)
protected function aggregateDataTableRecord($name, $columnsAggregationOperation = null, $columnsToRenameAfterAggregation = null)
{
$table = new DataTable();
if (!empty($columnAggregationOperations)) {
$table->setMetadata(DataTable::COLUMN_AGGREGATION_OPS_METADATA_NAME, $columnAggregationOperations);
}

$data = $this->getArchive()->getDataTableExpanded($name, $idSubTable = null, $depth = null, $addMetadataSubtableId = false);
if ($data instanceof DataTable\Map) {
// as $date => $tableToSum
foreach ($data->getDataTables() as $tableToSum) {
$table->addDataTable($tableToSum);
}
} else {
$table->addDataTable($data);
}

if (is_null($invalidSummedColumnNameToRenamedName)) {
$invalidSummedColumnNameToRenamedName = self::$invalidSummedColumnNameToRenamedName;
}
foreach ($invalidSummedColumnNameToRenamedName as $oldName => $newName) {
$table->renameColumn($oldName, $newName);
}
$dataTable = $this->getArchive()->getDataTableExpanded($name, $idSubTable = null, $depth = null, $addMetadataSubtableId = false);
$table = $this->getAggregatedDataTableMap($dataTable, $columnsAggregationOperation, $columnsToRenameAfterAggregation);
return $table;
}

Expand All @@ -388,64 +361,9 @@ protected function getOperationForColumns($columns, $defaultOperation)
return $operationForColumn;
}

protected function aggregateDataArray(array $data, array $operationForColumn)
{
$results = array();
foreach ($data as $row) {
if (!is_array($row)) {
// this is not a data array to aggregate
return $data;
}
foreach ($row as $name => $value) {
$operation = $operationForColumn[$name];
switch ($operation) {
case 'sum':
if (!isset($results[$name])) {
$results[$name] = 0;
}
$results[$name] += $value;
break;

case 'max':
if (!isset($results[$name])) {
$results[$name] = 0;
}
$results[$name] = max($results[$name], $value);
break;

case 'min':
if (!isset($results[$name])) {
$results[$name] = $value;
}
$results[$name] = min($results[$name], $value);
break;

case false:
// do nothing if the operation is not known (eg. nb_uniq_visitors should be not be aggregated)
break;

default:
throw new Exception("Operation not applicable.");
break;
}
}
}
return $results;
}

protected function defaultColumnsToZero($columns, $results)
{
foreach ($columns as $name) {
if (!isset($results[$name])) {
$results[$name] = 0;
}
}
return $results;
}

protected function enrichWithUniqueVisitorsMetric(&$results)
{
if (array_key_exists('nb_uniq_visitors', $results)) {
if (isset($results['nb_uniq_visitors'])) {
if (SettingsPiwik::isUniqueVisitorsEnabled($this->getParams()->getPeriod()->getLabel())) {
$results['nb_uniq_visitors'] = (float)$this->computeNbUniqVisitors();
} else {
Expand All @@ -462,9 +380,6 @@ protected function guessOperationForColumn($column)
if (strpos($column, 'min_') === 0) {
return 'min';
}
if ($column === 'nb_uniq_visitors') {
return false;
}
return 'sum';
}

Expand All @@ -483,4 +398,81 @@ protected function computeNbUniqVisitors()
$data = $query->fetch();
return $data[Metrics::INDEX_NB_UNIQ_VISITORS];
}

/**
* If the DataTable is a Map, sums all DataTable in the map and return the DataTable.
*
*
* @param $data DataTable|DataTable\Map
* @param $columnsToRenameAfterAggregation array
* @return DataTable
*/
protected function getAggregatedDataTableMap($data, $columnsAggregationOperation, $columnsToRenameAfterAggregation = null)
{
$table = new DataTable();
if (!empty($columnsAggregationOperation)) {
$table->setMetadata(DataTable::COLUMN_AGGREGATION_OPS_METADATA_NAME, $columnsAggregationOperation);
}
if ($data instanceof DataTable\Map) {
// as $date => $tableToSum
$this->aggregatedDataTableMapsAsOne($data, $table);
} else {
$table->addDataTable($data);
}

// Rename columns after aggregation
if (is_null($columnsToRenameAfterAggregation)) {
$columnsToRenameAfterAggregation = self::$columnsToRenameAfterAggregation;
}
foreach ($columnsToRenameAfterAggregation as $oldName => $newName) {
$table->renameColumn($oldName, $newName);
}
return $table;
}

/**
* Aggregates the DataTable\Map into the destination $aggregated
* @param $map
* @param $aggregated
*/
protected function aggregatedDataTableMapsAsOne(Map $map, DataTable $aggregated)
{
foreach ($map->getDataTables() as $tableToAggregate) {
if($tableToAggregate instanceof Map) {
$this->aggregatedDataTableMapsAsOne($tableToAggregate, $aggregated);
} else {
$aggregated->addDataTable($tableToAggregate);
}
}
}

protected function getAggregatedNumericMetrics($columns, $operationToApply)
{
if (!is_array($columns)) {
$columns = array($columns);
}
$operationForColumn = $this->getOperationForColumns($columns, $operationToApply);

$dataTable = $this->getArchive()->getDataTableFromNumeric($columns);

$results = $this->getAggregatedDataTableMap($dataTable, $operationForColumn);

if ($results->getRowsCount() > 1) {
throw new Exception("A DataTable is an unexpected state:" . var_export($results, true));
}
$rowMetrics = $results->getFirstRow();
if ($rowMetrics === false) {
$metrics = array();
} else {
$metrics = $rowMetrics->getColumns();
}
$this->enrichWithUniqueVisitorsMetric($metrics);

foreach ($columns as $name) {
if (!isset($metrics[$name])) {
$metrics[$name] = 0;
}
}
return $metrics;
}
}
28 changes: 26 additions & 2 deletions core/ArchiveProcessor/Parameters.php
View file
Expand Up @@ -72,6 +72,28 @@ public function getPeriod()
return $this->period;
}

/**
* Returns the array of Period which make up this archive.
*
* @return \Piwik\Period[]
*/
public function getSubPeriods()
{
if($this->getPeriod()->getLabel() == 'day') {
return array( $this->getPeriod() );
}
return $this->getPeriod()->getSubperiods();
}

/**
* @return array
*/
public function getIdSites()
{
$idSite = $this->getSite()->getId();
return array($idSite);
}

/**
* Returns the site we are computing statistics for.
*
Expand Down Expand Up @@ -117,9 +139,11 @@ public function getDateStart()
/**
* @return bool
*/
public function isDayArchive()
public function isSingleSiteDayArchive()
{
return $this->getPeriod()->getLabel() == 'day';
$oneSite = count($this->getIdSites()) == 1;
$oneDay = $this->getPeriod()->getLabel() == 'day';
return $oneDay && $oneSite;
}

public function logStatusDebug($isTemporary)
Expand Down

0 comments on commit f197294

Please sign in to comment.