refs #1700 performance analytics

 * adding avg_time_generation to Actions.get + integration tests
 * adding sparkline for average generation time to Visitors > Overview
 * changing number formatting to 0.XXs instead of XXXms + test cases
 * tooltip for reports with avg. generation time: "average based on X hit(s)"
 * log import: support generation_time_milli (not only generation_time_micro)
 * example for importing generation time from logs in read me

core/Piwik.php

@@ -1417,9 +1417,9 @@ static public function getPrettyTimeFromSeconds($numberOfSeconds, $displayTimeAs
             $minutes = floor(($reminder = ($numberOfSeconds - $hours * 3600)) / 60);
             $seconds = floor($reminder - $minutes * 60);
             $time = sprintf("%02s", $hours) . ':' . sprintf("%02s", $minutes) . ':' . sprintf("%02s", $seconds);
-            $milliSeconds = ($numberOfSeconds * 1000) % 1000;
-            if ($milliSeconds) {
-                $time .= '.' . sprintf("%03s", $milliSeconds);
+            $centiSeconds = ($numberOfSeconds * 100) % 100;
+            if ($centiSeconds) {
+                $time .= '.' . sprintf("%02s", $centiSeconds);
             }
             return $time;
         }
@@ -1435,8 +1435,7 @@ static public function getPrettyTimeFromSeconds($numberOfSeconds, $displayTimeAs
         $minutes = floor($minusDaysAndHours / 60);
 
         $seconds = $minusDaysAndHours - $minutes * 60;
-
-		$milliSeconds = ($numberOfSeconds * 1000) % 1000;
+		$seconds = round($seconds, 2);
 
         if ($years > 0) {
             $return = sprintf(Piwik_Translate('General_YearsDays'), $years, $days);
@@ -1446,9 +1445,7 @@ static public function getPrettyTimeFromSeconds($numberOfSeconds, $displayTimeAs
             $return = sprintf(Piwik_Translate('General_HoursMinutes'), $hours, $minutes);
         } elseif ($minutes > 0) {
             $return = sprintf(Piwik_Translate('General_MinutesSeconds'), $minutes, $seconds);
-		} elseif ($milliSeconds > 0 && $seconds < 1) {
-			$return = sprintf(Piwik_Translate('General_Milliseconds'), $milliSeconds);
-        } else {
+		} else {
             $return = sprintf(Piwik_Translate('General_Seconds'), $seconds);
         }
         if ($isHtml) {

lang/en.php

@@ -191,7 +191,6 @@
 	'General_HoursMinutes' => '%1$s hours %2$s min',
 	'General_MinutesSeconds' => '%1$s min %2$ss',
 	'General_Seconds' => '%ss',
-	'General_Milliseconds' => '%sms',
 	'General_Save' => 'Save',
 	'General_Faq' => 'FAQ',
 	'General_ForExampleShort' => 'eg.',
@@ -439,6 +438,7 @@
 	'Actions_SiteSearchCategories2' => 'For example, Ecommerce websites typically have a "Category" selector so that visitors can restrict their searches to all products in a specific Category.',
 	'Actions_SiteSearchKeywordsNoResultDocumentation' => 'This report lists the Search Keywords that did not return any Search result: maybe the search engine algorithm can be improved, or maybe your visitors are looking for content that is not (yet) on your website?',
 	'Actions_SiteSearchFollowingPagesDoc' => 'When visitors search on your website, they are looking for a particular page, content, product, or service. This report lists the pages that were clicked the most after an internal search. In other words, the list of pages the most searched for by visitors already on your website.',
+	'Actions_AvgGenerationTimeTooltip' => 'Average based on %s hit(s)',
 	'AnonymizeIP_PluginDescription' => 'Anonymize the last byte(s) of visitors IP addresses to comply with your local privacy laws/guidelines.',
 	'API_PluginDescription' => 'All the data in Piwik is available through simple APIs. This plugin is the web service entry point, that you can call to get your Web Analytics data in xml, json, php, csv, etc.',
 	'API_QuickDocumentationTitle' => 'API quick documentation',
@@ -1940,6 +1940,7 @@
 	'VisitsSummary_MaxNbActions' => '%s max actions in one visit',
 	'VisitsSummary_NbActionsPerVisit' => '%s actions (page views, downloads, outlinks and internal site searches) per visit',
 	'VisitsSummary_NbVisitsBounced' => '%s visits have bounced (left the website after one page)',
+	'VisitsSummary_AverageGenerationTime' => '%s average generation time',
 	'VisitsSummary_GenerateTime' => '%s seconds to generate the page',
 	'VisitsSummary_GenerateQueries' => '%s queries executed',
 	'VisitsSummary_WidgetLastVisits' => 'Visits Over Time',

misc/log-analytics/README.md

@@ -175,3 +175,13 @@ This log format can be specified for nginx access logs to capture multiple virtu
 * access_log /PATH/TO/access.log vhosts;
 
 When executing import_logs.py specify the "common_complete" format.
+
+
+## Import Generation Time
+
+Apache can log the generation time in microseconds using %D in the LogFormat.
+This metric can be imported using a custom log format in this script, which means that you have to specify a --log-format-regex parameter that contains the group generation_time_micro.  
+
+Here's an example:
+Apache LogFormat "%h %l %u %t \"%r\" %>s %b %D"
+--log-format-regex="(?P<ip>\S+) \S+ \S+ \[(?P<date>.*?) (?P<timezone>.*?)\] \"\S+ (?P<path>.*?) \S+\" (?P<status>\S+) (?P<length>\S+) (?P<generation_time_micro>\S+)"

misc/log-analytics/import_logs.py

@@ -1437,11 +1437,14 @@ def invalid_line(line, reason):
             except (ValueError, IndexError):
                 # Some lines or formats don't have a length (e.g. 304 redirects, IIS logs)
                 hit.length = 0
-                
+
             try:
-			    hit.generation_time_milli = int(match.group('generation_time_micro')) / 1000
+                hit.generation_time_milli = int(match.group('generation_time_milli'))
             except IndexError:
-                hit.generation_time_milli = 0
+                try:
+                    hit.generation_time_milli = int(match.group('generation_time_micro')) / 1000
+                except IndexError:
+                    hit.generation_time_milli = 0
 
             if config.options.log_hostname:
                 hit.host = config.options.log_hostname

plugins/Actions/API.php

@@ -73,14 +73,15 @@ public function get($idSite, $period, $date, $segment = false, $columns = false)
         $archive = Piwik_Archive::build($idSite, $period, $date, $segment);
 
         $metrics = array(
-            'Actions_nb_pageviews'      => 'nb_pageviews',
-            'Actions_nb_uniq_pageviews' => 'nb_uniq_pageviews',
-            'Actions_nb_downloads'      => 'nb_downloads',
-            'Actions_nb_uniq_downloads' => 'nb_uniq_downloads',
-            'Actions_nb_outlinks'       => 'nb_outlinks',
-            'Actions_nb_uniq_outlinks'  => 'nb_uniq_outlinks',
-            'Actions_nb_searches'       => 'nb_searches',
-            'Actions_nb_keywords'       => 'nb_keywords',
+            'Actions_nb_pageviews'        => 'nb_pageviews',
+            'Actions_nb_uniq_pageviews'   => 'nb_uniq_pageviews',
+            'Actions_nb_downloads'        => 'nb_downloads',
+            'Actions_nb_uniq_downloads'   => 'nb_uniq_downloads',
+            'Actions_nb_outlinks'         => 'nb_outlinks',
+            'Actions_nb_uniq_outlinks'    => 'nb_uniq_outlinks',
+            'Actions_nb_searches'         => 'nb_searches',
+            'Actions_nb_keywords'         => 'nb_keywords',
+			'Actions_avg_time_generation' => 'avg_time_generation'
         );
 
         // get requested columns
@@ -95,16 +96,38 @@ public function get($idSite, $period, $date, $segment = false, $columns = false)
                 $columns[$i] = $fullColumn;
                 $nameReplace[$fullColumn] = $column;
             }
+
+			if (false !== ($avgGenerationTimeRequested = array_search('Actions_avg_time_generation', $columns))) {
+				unset($columns[$avgGenerationTimeRequested]);
+				$avgGenerationTimeRequested = true;
+			}
         } else {
             // get all columns
+			unset($metrics['Actions_avg_time_generation']);
             $columns = array_keys($metrics);
             $nameReplace = & $metrics;
+			$avgGenerationTimeRequested = true;
         }
-
+
+		if ($avgGenerationTimeRequested) {
+			$tempColumns[] = 'Actions_sum_time_generation';
+			$tempColumns[] = 'Actions_nb_hits_with_time_generation';
+			$nameReplace['Actions_sum_time_generation'] = 'sum_time_generation';
+			$nameReplace['Actions_nb_hits_with_time_generation'] = 'nb_hits_with_time_generation';
+			$columns = array_merge($columns, $tempColumns);
+			$columns = array_unique($columns);
+		}
+
         $table = $archive->getDataTableFromNumeric($columns);
 
         // replace labels (remove Actions_)
         $table->filter('ReplaceColumnNames', array($nameReplace));
+
+		// compute avg generation time
+		if ($avgGenerationTimeRequested) {
+			$table->filter('ColumnCallbackAddColumnQuotient', array('avg_time_generation', 'sum_time_generation', 'nb_hits_with_time_generation', 3));
+			$table->deleteColumns(array('sum_time_generation', 'nb_hits_with_time_generation'));
+		}
 
         return $table;
     }

plugins/Actions/Actions.php

@@ -185,23 +185,25 @@ public function getReportMetadata($notification)
             'module'               => 'Actions',
             'action'               => 'get',
             'metrics'              => array(
-                'nb_pageviews'      => Piwik_Translate('General_ColumnPageviews'),
-                'nb_uniq_pageviews' => Piwik_Translate('General_ColumnUniquePageviews'),
-                'nb_downloads'      => Piwik_Translate('Actions_ColumnDownloads'),
-                'nb_uniq_downloads' => Piwik_Translate('Actions_ColumnUniqueDownloads'),
-                'nb_outlinks'       => Piwik_Translate('Actions_ColumnOutlinks'),
-                'nb_uniq_outlinks'  => Piwik_Translate('Actions_ColumnUniqueOutlinks'),
-                'nb_searches'       => Piwik_Translate('Actions_ColumnSearches'),
-                'nb_keywords'       => Piwik_Translate('Actions_ColumnSiteSearchKeywords'),
+                'nb_pageviews'        => Piwik_Translate('General_ColumnPageviews'),
+                'nb_uniq_pageviews'   => Piwik_Translate('General_ColumnUniquePageviews'),
+                'nb_downloads'        => Piwik_Translate('Actions_ColumnDownloads'),
+                'nb_uniq_downloads'   => Piwik_Translate('Actions_ColumnUniqueDownloads'),
+                'nb_outlinks'         => Piwik_Translate('Actions_ColumnOutlinks'),
+                'nb_uniq_outlinks'    => Piwik_Translate('Actions_ColumnUniqueOutlinks'),
+                'nb_searches'         => Piwik_Translate('Actions_ColumnSearches'),
+                'nb_keywords'         => Piwik_Translate('Actions_ColumnSiteSearchKeywords'),
+				'avg_time_generation' => Piwik_Translate('General_ColumnAverageGenerationTime'),
             ),
             'metricsDocumentation' => array(
-                'nb_pageviews'      => Piwik_Translate('General_ColumnPageviewsDocumentation'),
-                'nb_uniq_pageviews' => Piwik_Translate('General_ColumnUniquePageviewsDocumentation'),
-                'nb_downloads'      => Piwik_Translate('Actions_ColumnClicksDocumentation'),
-                'nb_uniq_downloads' => Piwik_Translate('Actions_ColumnUniqueClicksDocumentation'),
-                'nb_outlinks'       => Piwik_Translate('Actions_ColumnClicksDocumentation'),
-                'nb_uniq_outlinks'  => Piwik_Translate('Actions_ColumnUniqueClicksDocumentation'),
-                'nb_searches'       => Piwik_Translate('Actions_ColumnSearchesDocumentation'),
+                'nb_pageviews'        => Piwik_Translate('General_ColumnPageviewsDocumentation'),
+                'nb_uniq_pageviews'   => Piwik_Translate('General_ColumnUniquePageviewsDocumentation'),
+                'nb_downloads'        => Piwik_Translate('Actions_ColumnClicksDocumentation'),
+                'nb_uniq_downloads'   => Piwik_Translate('Actions_ColumnUniqueClicksDocumentation'),
+                'nb_outlinks'         => Piwik_Translate('Actions_ColumnClicksDocumentation'),
+                'nb_uniq_outlinks'    => Piwik_Translate('Actions_ColumnUniqueClicksDocumentation'),
+                'nb_searches'         => Piwik_Translate('Actions_ColumnSearchesDocumentation'),
+				'avg_time_generation' => Piwik_Translate('General_ColumnAverageGenerationTimeDocumentation'),
 //				'nb_keywords' => Piwik_Translate('Actions_ColumnSiteSearchKeywords'),
             ),
             'processedMetrics'     => false,

plugins/Actions/Archiving.php

@@ -74,6 +74,8 @@ public function archivePeriod(Piwik_ArchiveProcessing $archiveProcessing)
                                                          'Actions_nb_outlinks',
                                                          'Actions_nb_uniq_outlinks',
                                                          'Actions_nb_searches',
+                                                         'Actions_sum_time_generation',
+                                                         'Actions_nb_hits_with_time_generation',
                                                     ));
 
         // Unique Keywords can't be summed, instead we take the RowsCount() of the keyword table
@@ -359,6 +361,8 @@ protected function archiveDayRecordInDatabase($archiveProcessing)
         $archiveProcessing->insertBlobRecord('Actions_actions_url', $s);
         $archiveProcessing->insertNumericRecord('Actions_nb_pageviews', array_sum($dataTable->getColumn(Piwik_Archive::INDEX_PAGE_NB_HITS)));
         $archiveProcessing->insertNumericRecord('Actions_nb_uniq_pageviews', array_sum($dataTable->getColumn(Piwik_Archive::INDEX_NB_VISITS)));
+		$archiveProcessing->insertNumericRecord('Actions_sum_time_generation', array_sum($dataTable->getColumn(Piwik_Archive::INDEX_PAGE_SUM_TIME_GENERATION)));
+		$archiveProcessing->insertNumericRecord('Actions_nb_hits_with_time_generation', array_sum($dataTable->getColumn(Piwik_Archive::INDEX_PAGE_NB_HITS_WITH_TIME_GENERATION)));
         destroy($dataTable);
 
         $dataTable = $this->actionsTablesByType[Piwik_Tracker_Action::TYPE_DOWNLOAD];

plugins/Actions/Controller.php

@@ -385,9 +385,14 @@ protected function configureViewActions($view, $doSetTranslations = true)
             $view->setColumnTranslation('bounce_rate', Piwik_Translate('General_ColumnBounceRate'));
             $view->setColumnTranslation('exit_rate', Piwik_Translate('General_ColumnExitRate'));
             $view->setColumnTranslation('avg_time_generation', Piwik_Translate('General_ColumnAverageGenerationTime'));
-            $view->queueFilter('ColumnCallbackReplace', array('avg_time_on_page', array('Piwik', 'getPrettyTimeFromSeconds')));
-            $view->queueFilter('ColumnCallbackReplace', array('avg_time_generation',
-                                                              create_function('$averageTimeOnSite', 'return $averageTimeOnSite ? Piwik::getPrettyTimeFromSeconds($averageTimeOnSite, true, true, false) : "-";')));
+
+			$view->queueFilter('ColumnCallbackReplace', array('avg_time_on_page', array('Piwik', 'getPrettyTimeFromSeconds')));
+
+			$avgTimeCallback = create_function('$time', 'return $time ? Piwik::getPrettyTimeFromSeconds($time, true, true, false) : "-";');
+            $view->queueFilter('ColumnCallbackReplace', array('avg_time_generation', $avgTimeCallback));
+
+			$tooltipCallback = create_function('$hits', 'return $hits ? Piwik_Translate("Actions_AvgGenerationTimeTooltip", $hits) : false;');
+			$view->queueFilter('ColumnCallbackAddMetadata', array('nb_hits_with_time_generation', 'avg_time_generation_tooltip', $tooltipCallback));
         }
 
         if (Piwik_Common::getRequestVar('enable_filter_excludelowpop', '0', 'string') != '0') {

plugins/CoreHome/templates/datatable.css

@@ -803,3 +803,7 @@ a.tableConfigurationIcon.highlighted {
 .tableConfiguration div.configItem span.action {
     color: #255792;
 }
+
+table.dataTable span.cell-tooltip {
+	cursor: default;
+}

plugins/CoreHome/templates/datatable.js

@@ -223,6 +223,7 @@ dataTable.prototype =
         self.handleColumnDocumentation(domElem);
         self.handleReportDocumentation(domElem);
         self.handleRowActions(domElem);
+		self.handleCellTooltips(domElem);
         self.handleRelatedReports(domElem);
         self.handleTriggeredEvents(domElem);
     },
@@ -1312,6 +1313,19 @@ dataTable.prototype =
     handleRowActions: function (domElem) {
         this.doHandleRowActions(domElem.find('table > tbody > tr'));
     },
+
+	handleCellTooltips: function(domElem) {
+		domElem.find('span.cell-tooltip').tooltip({
+			track: true,
+			items: 'span',
+			content: function() {
+				return $(this).data('tooltip');
+			},
+			show: false,
+			hide: false,
+			tooltipClass: 'small'
+		});
+	},
 
     handleRelatedReports: function (domElem) {
         var self = this,
@@ -1576,6 +1590,7 @@ actionDataTable.prototype =
     notifyWidgetParametersChange: dataTable.prototype.notifyWidgetParametersChange,
     handleRelatedReports: dataTable.prototype.handleRelatedReports,
     handleTriggeredEvents: dataTable.prototype.handleTriggeredEvents,
+	handleCellTooltips: dataTable.prototype.handleCellTooltips,
     _findReportHeader: dataTable.prototype._findReportHeader,
 
     //initialisation of the actionDataTable
@@ -1627,6 +1642,7 @@ actionDataTable.prototype =
         self.handleReportDocumentation(domElem);
         self.handleRelatedReports(domElem);
         self.handleTriggeredEvents(domElem);
+		self.handleCellTooltips(domElem);
     },
 
     //see dataTable::applyCosmetics

plugins/CoreHome/templates/datatable_cell.tpl

@@ -1,3 +1,5 @@
+{assign var="tooltipIndex" value=$column|cat:"_tooltip"}
+{if isset($row.metadata[$tooltipIndex])}<span class="cell-tooltip" data-tooltip="{$row.metadata[$tooltipIndex]|escape:'html'}">{/if}
 {if !$row.idsubdatatable && $column=='label' && !empty($row.metadata.url)}
 <a target="_blank" href='{if !in_array(substr($row.metadata.url,0,4), array('http','ftp:'))}http://{/if}{$row.metadata.url|escape:'html'}'>
     {if empty($row.metadata.logo)}
@@ -16,3 +18,4 @@
     {if !$row.idsubdatatable && $column=='label' && !empty($row.metadata.url)}
 </a>
 {/if}
+{if isset($row.metadata[$tooltipIndex])}</span>{/if}

plugins/VisitsSummary/Controller.php

@@ -116,6 +116,7 @@ protected function setSparklinesAndNumbers($view)
         $view->urlSparklineMaxActions = $this->getUrlSparkline('getEvolutionGraph', array('columns' => array('max_actions')));
         $view->urlSparklineActionsPerVisit = $this->getUrlSparkline('getEvolutionGraph', array('columns' => array('nb_actions_per_visit')));
         $view->urlSparklineBounceRate = $this->getUrlSparkline('getEvolutionGraph', array('columns' => array('bounce_rate')));
+		$view->urlSparklineAvgGenerationTime = $this->getUrlSparkline('getEvolutionGraph', array('columns' => array('avg_time_generation')));
 
         $idSite = Piwik_Common::getRequestVar('idSite');
         $displaySiteSearch = Piwik_Site::isSiteSearchEnabledFor($idSite);
@@ -145,6 +146,7 @@ protected function setSparklinesAndNumbers($view)
         $view->bounceRate = Piwik::getPercentageSafe($nbBouncedVisits, $nbVisits);
         $view->maxActions = (int)$dataRow->getColumn('max_actions');
         $view->nbActionsPerVisit = $dataRow->getColumn('nb_actions_per_visit');
+		$view->averageGenerationTime = $dataActionsRow->getColumn('avg_time_generation');
 
         if ($displaySiteSearch) {
             $view->nbSearches = (int)$dataActionsRow->getColumn('nb_searches');

plugins/VisitsSummary/templates/sparklines.tpl

@@ -17,6 +17,11 @@
         {sparkline src=$urlSparklineActionsPerVisit}
         {'VisitsSummary_NbActionsPerVisit'|translate:"<strong>$nbActionsPerVisit</strong>"}
     </div>
+	<div class="sparkline">
+		{sparkline src=$urlSparklineAvgGenerationTime}
+		{assign var=averageGenerationTime value=$averageGenerationTime|sumtime}
+		{'VisitsSummary_AverageGenerationTime'|translate:"<strong>$averageGenerationTime</strong>"}
+	</div>
 </div>
 
 <div id='rightcolumn'>

tests/PHPUnit/Core/PiwikTest.php

@@ -88,11 +88,13 @@ public function getPrettyTimeFromSecondsData()
             array(86400 + 3600 * 10, array('1 days 10 hours', '34:00:00')),
             array(86400 * 365, array('365 days 0 hours', '8760:00:00')),
             array((86400 * (365.25 + 10)), array('1 years 10 days', '9006:00:00')),
-            array(1.342, array('1.342s', '00:00:01.342')),
-            array(.342, array('342ms', '00:00:00.342')),
-			array(.02, array('20ms', '00:00:00.020')),
-			array(1.002, array('1.002s', '00:00:01.002')),
-			array(122.1, array('2 min 2.1s', '00:02:02.100'))
+            array(1.342, array('1.34s', '00:00:01.34')),
+            array(.342, array('0.34s', '00:00:00.34')),
+			array(.02, array('0.02s', '00:00:00.02')),
+			array(1.002, array('1s', '00:00:01')),
+			array(1.02, array('1.02s', '00:00:01.02')),
+			array(1.2, array('1.2s', '00:00:01.20')),
+			array(122.1, array('2 min 2.1s', '00:02:02.10'))
         );
     }