Permalink
Browse files

visitor log improvements

 * use a limit for the number of actions that are loaded for every visit. without a limit, the web server might crash (e.g. when there are 100K+ page views in one visit after a log import)
 * regular pagination in visitor log. (a) performance improvement: use indexes in the select query. (b) bug fix: the old logic was incorrect. in one setup, there are 10K+ visits but the visitor log only showed one full page and one page with one visit.
 * removed reference to trunk from tests read me
  • Loading branch information...
1 parent 5572724 commit 3ac47e89eb7cc6024cffad435e9a6ef2a29488ff @BeezyT BeezyT committed Mar 20, 2013
View
@@ -248,6 +248,9 @@
; amount of actions, referrers or custom variable name/value pairs.
archiving_ranking_query_row_limit = 50000
+; maximum number of actions that is shown in the visitor log for each visitor
+visitor_log_maximum_actions_per_visit = 500
+
; by default, the real time Live! widget will update every 5 seconds and refresh with new visits/actions/etc.
; you can change the timeout so the widget refreshes more often, or not as frequently
live_widget_refresh_after_seconds = 5
View
@@ -772,7 +772,9 @@ protected function getJavascriptVariablesToSet()
if($this->dataTable &&
// Piwik_DataTable_Array doesn't have the method
- !($this->dataTable instanceof Piwik_DataTable_Array))
+ !($this->dataTable instanceof Piwik_DataTable_Array)
+ && empty($javascriptVariablesToSet['totalRows'])
+ )
{
$javascriptVariablesToSet['totalRows'] = $this->dataTable->getRowsCountBeforeLimitFilter();
}
@@ -279,9 +279,6 @@ dataTable.prototype =
{
params.filter_limit = value;
params.filter_offset = 0;
-
- // Hack for Visitor Log to not pass the maxIdVisit parameter when limit is changed
- delete params.maxIdVisit;
};
}
View
@@ -100,8 +100,7 @@ public function getCounters($idSite, $lastMinutes, $segment = false)
*/
public function getLastVisitsForVisitor( $visitorId, $idSite, $filter_limit = 10 )
{
- Piwik::checkUserHasViewAccess($idSite);
- $visitorDetails = $this->loadLastVisitorDetailsFromDatabase($idSite, $period = false, $date = false, $segment = false, $filter_limit, $maxIdVisit = false, $visitorId);
+ $visitorDetails = $this->loadLastVisitorDetailsFromDatabase($idSite, $period = false, $date = false, $segment = false, $filter_limit, $filter_offset = false, $visitorId);
$table = $this->getCleanedVisitorsFromDetails($visitorDetails, $idSite);
return $table;
}
@@ -114,21 +113,22 @@ public function getLastVisitsForVisitor( $visitorId, $idSite, $filter_limit = 10
* @param bool|string $period Period to restrict to when looking at the logs
* @param bool|string $date Date to restrict to
* @param bool|int $segment (optional) Number of visits rows to return
- * @param bool|int $filter_limit (optional)
- * @param bool|int $maxIdVisit (optional) Maximum idvisit to restrict the query to (useful when paginating)
+ * @param bool|int $filter_limit (optional) Only return X visits
+ * @param bool|int $filter_offset (optional) Skip the first X visits (useful when paginating)
* @param bool|int $minTimestamp (optional) Minimum timestamp to restrict the query to (useful when paginating or refreshing visits)
*
* @return Piwik_DataTable
*/
- public function getLastVisitsDetails( $idSite, $period, $date, $segment = false, $filter_limit = false, $maxIdVisit = false, $minTimestamp = false )
+ public function getLastVisitsDetails( $idSite, $period, $date, $segment = false, $filter_limit = false, $filter_offset = false, $minTimestamp = false )
{
if(empty($filter_limit))
{
$filter_limit = 10;
}
Piwik::checkUserHasViewAccess($idSite);
- $visitorDetails = $this->loadLastVisitorDetailsFromDatabase($idSite, $period, $date, $segment, $filter_limit, $maxIdVisit, $visitorId = false, $minTimestamp);
+ $visitorDetails = $this->loadLastVisitorDetailsFromDatabase($idSite, $period, $date, $segment, $filter_limit, $filter_offset, $visitorId = false, $minTimestamp);
$dataTable = $this->getCleanedVisitorsFromDetails($visitorDetails, $idSite);
+
return $dataTable;
}
@@ -138,7 +138,7 @@ public function getLastVisitsDetails( $idSite, $period, $date, $segment = false,
public function getLastVisits( $idSite, $filter_limit = 10, $minTimestamp = false )
{
- return $this->getLastVisitsDetails($idSite, $period = false, $date = false, $segment = false, $filter_limit, $maxIdVisit = false, $minTimestamp );
+ return $this->getLastVisitsDetails($idSite, $period = false, $date = false, $segment = false, $filter_limit, $filter_offset = false, $minTimestamp );
}
/**
@@ -150,6 +150,8 @@ public function getLastVisits( $idSite, $filter_limit = 10, $minTimestamp = fals
*/
private function getCleanedVisitorsFromDetails($visitorDetails, $idSite)
{
+ $actionsLimit = Piwik_Config::getInstance()->General['visitor_log_maximum_actions_per_visit'];
+
$table = new Piwik_DataTable();
$site = new Piwik_Site($idSite);
@@ -198,6 +200,7 @@ private function getCleanedVisitorsFromDetails($visitorDetails, $idSite)
LEFT JOIN " .Piwik_Common::prefixTable('log_action')." AS log_action_title
ON log_link_visit_action.idaction_name = log_action_title.idaction
WHERE log_link_visit_action.idvisit = ?
+ LIMIT 0, $actionsLimit
";
$actionDetails = Piwik_FetchAll($sql, array($idvisit));
@@ -253,6 +256,7 @@ private function getCleanedVisitorsFromDetails($visitorDetails, $idSite)
AND goal.deleted = 0
WHERE log_conversion.idvisit = ?
AND log_conversion.idgoal > 0
+ LIMIT 0, $actionsLimit
";
$goalDetails = Piwik_FetchAll($sql, array($idvisit));
@@ -269,7 +273,8 @@ private function getCleanedVisitorsFromDetails($visitorDetails, $idSite)
log_conversion.server_time as serverTimePretty
FROM ".Piwik_Common::prefixTable('log_conversion')." AS log_conversion
WHERE idvisit = ?
- AND idgoal <= ".Piwik_Tracker_GoalManager::IDGOAL_ORDER;
+ AND idgoal <= ".Piwik_Tracker_GoalManager::IDGOAL_ORDER."
+ LIMIT 0, $actionsLimit";
$ecommerceDetails = Piwik_FetchAll($sql, array($idvisit));
foreach($ecommerceDetails as &$ecommerceDetail)
@@ -316,6 +321,7 @@ private function getCleanedVisitorsFromDetails($visitorDetails, $idSite)
WHERE idvisit = ?
AND idorder = ?
AND deleted = 0
+ LIMIT 0, $actionsLimit
";
$bind = array($idvisit, isset($ecommerceConversion['orderId'])
? $ecommerceConversion['orderId']
@@ -400,12 +406,17 @@ private function sortByServerTime($a, $b)
: 1 );
}
- private function loadLastVisitorDetailsFromDatabase($idSite, $period = false, $date = false, $segment = false, $filter_limit = false, $maxIdVisit = false, $visitorId = false, $minTimestamp = false)
+ private function loadLastVisitorDetailsFromDatabase($idSite, $period = false, $date = false, $segment = false, $filter_limit = false, $filter_offset = false, $visitorId = false, $minTimestamp = false)
{
if(empty($filter_limit))
{
$filter_limit = 100;
}
+ if(empty($filter_offset))
+ {
+ $filter_offset = 0;
+ }
+
$where = $whereBind = array();
$where[] = "log_visit.idsite = ? ";
$whereBind[] = $idSite;
@@ -417,14 +428,6 @@ private function loadLastVisitorDetailsFromDatabase($idSite, $period = false, $d
$whereBind[] = @Piwik_Common::hex2bin($visitorId);
}
- if(!empty($maxIdVisit))
- {
- $where[] = "log_visit.idvisit < ? ";
- $whereBind[] = $maxIdVisit;
- $orderBy = "idvisit DESC";
- $orderByParent = "sub.idvisit DESC";
- }
-
if(!empty($minTimestamp))
{
$where[] = "log_visit.visit_last_action_time > ? ";
@@ -433,7 +436,7 @@ private function loadLastVisitorDetailsFromDatabase($idSite, $period = false, $d
// If no other filter, only look at the last 24 hours of stats
if(empty($visitorId)
- && empty($maxIdVisit)
+ && empty($filter_offset)
&& empty($period)
&& empty($date))
{
@@ -500,7 +503,7 @@ private function loadLastVisitorDetailsFromDatabase($idSite, $period = false, $d
$from = "log_visit";
$subQuery = $segment->getSelectQuery($select, $from, $where, $whereBind, $orderBy);
- $sqlLimit = $filter_limit >= 1 ? " LIMIT ".(int)$filter_limit : "";
+ $sqlLimit = $filter_limit >= 1 ? " LIMIT ".$filter_offset.", ".(int)$filter_limit : "";
// Group by idvisit so that a visitor converting 2 goals only appears once
$sql = "
@@ -49,12 +49,6 @@ private function render($view, $fetch)
public function getVisitorLog($fetch = false)
{
- // If previous=1 is set, user clicked previous
- // we can't deal with previous so we force display of the first page
- if(Piwik_Common::getRequestVar('previous', 0, 'int') == 1) {
- $_GET['maxIdVisit'] = '';
- }
-
$view = Piwik_ViewDataTable::factory();
$view->init( $this->pluginName,
__FUNCTION__,
@@ -83,9 +77,13 @@ public function getVisitorLog($fetch = false)
}
$view->setReportDocumentation(Piwik_Translate('Live_VisitorLogDocumentation', array('<br />', '<br />')));
- $view->setCustomParameter('dataTablePreviousIsFirst', 1);
+
+ // set a very high row count so that the next link in the footer of the data table is always shown
+ $view->setCustomParameter('totalRows', 10000000);
+
$view->setCustomParameter('filterEcommerce', Piwik_Common::getRequestVar('filterEcommerce', 0, 'int'));
$view->setCustomParameter('pageUrlNotDefined', Piwik_Translate('General_NotDefined', Piwik_Translate('Actions_ColumnPageURL')));
+
return $this->renderView($view, $fetch);
}
@@ -10,7 +10,7 @@
{capture assign='displayVisitorsInOwnColumn'}{if $isWidget}0{else}1{/if}{/capture}
<a graphid="VisitsSummarygetEvolutionGraph" name="evolutionGraph"></a>
-{assign var=maxIdVisit value=0}
+
{if isset($arrayDataTable.result) and $arrayDataTable.result == 'error'}
{$arrayDataTable.message}
{else}
@@ -39,9 +39,6 @@
<tbody>
{foreach from=$arrayDataTable item=visitor}
-{if $maxIdVisit == 0 || $visitor.columns.idVisit < $maxIdVisit}
-{assign var=maxIdVisit value=$visitor.columns.idVisit}
-{/if}
{capture assign='visitorColumnContent'}
&nbsp;<img src="{$visitor.columns.countryFlag}" title="{$visitor.columns.location|escape:'html'}, Provider {$visitor.columns.provider|escape:'html'}" />
@@ -257,13 +254,6 @@
</table>
{/if}
-{if count($arrayDataTable) == $javascriptVariablesToSet.filter_limit}
-{* We set a fake large rows count so that 'Next' paginate link is forced to display
- This is hard coded because the Visitor Log datatable is not fully loaded in memory,
- but needs to fetch only the N rows in the logs
- *}
-{php}$this->_tpl_vars['javascriptVariablesToSet']['totalRows'] = 100000; {/php}
-{/if}
{if $properties.show_footer}
{include file="CoreHome/templates/datatable_footer.tpl"}
{/if}
@@ -280,18 +270,7 @@ function Piwik_Live_LoadVisitorPopover(visitorId)
{rdelim}
$(document).ready(function(){ldelim}
-
- var dataTableVisitorLog = piwik.DataTableManager.getDataTableInstanceByReport('{$properties.uniqueId}');
- dataTableVisitorLog.param.maxIdVisit = {$maxIdVisit};
- {literal}
- function hidePreviousLink() {
- if (dataTableVisitorLog.param.previous == 1) {
- $('.dataTablePrevious').hide();
- dataTableVisitorLog.param.previous = 0;
- }
- }
- hidePreviousLink();
-
+ {literal}
// Replace duplicated page views by a NX count instead of using too much vertical space
$("ol.visitorLog").each(function () {
var prevelement;
View
@@ -141,7 +141,7 @@ which you can use to run PHPUnit tests in your browser.
### Starting VisualPHPUnit
-To load VisualPHPUnit point your browser to http://path/to/piwik/trunk/tests/lib/visualphpunit/.
+To load VisualPHPUnit point your browser to http://path/to/piwik/tests/lib/visualphpunit/.
VisualPHPUnit will already be configured for use with Piwik.
@@ -215,7 +215,7 @@ First, XHProf must be built (this guide assumes you're using a linux variant):
* Navigate to the XHProf extension directory.
- $ cd /path/to/piwik/trunk/tests/lib/xhprof-0.9.2/extension
+ $ cd /path/to/piwik/tests/lib/xhprof-0.9.2/extension
* Build XHProf.
@@ -228,7 +228,7 @@ First, XHProf must be built (this guide assumes you're using a linux variant):
```
[xhprof]
- extension=/path/to/piwik/trunk/tests/lib/xhprof-0.9.2/extension/modules/xhprof.so
+ extension=/path/to/piwik/tests/lib/xhprof-0.9.2/extension/modules/xhprof.so
xhprof.output_dir=/path/to/output/dir
```
@@ -242,7 +242,7 @@ is installed and act accordingly.
To use XHProf, first load VisualPHPUnit by pointing your browser to:
-http://path/to/piwik/trunk/tests/lib/visualphpunit/
+http://path/to/piwik/tests/lib/visualphpunit/
Select a test or get ready to run a benchmark. Make sure the 'Profile with XHProf' select
box is set to 'Yes' and click 'Run Tests'.

2 comments on commit 3ac47e8

Member

halfdan replied Mar 20, 2013

Very nice one!

Member

BeezyT replied Mar 20, 2013

Thanks... it took me while to figure out ;)
There's one thing left to improve, though. When a visitor views 100x the same page (which is strangely often the case when importing logs), we deliver 100 lis with the same URL in the visitor log table. Java Script will then aggregate the lis and add the "100x" label (in visitorLog.tpl). It would save lots of bandwith, memory and client hickups (because the JS aggregation is blocking the UI thread) if we would aggregate on the server side.
Is anybody interested in doing that?

Please sign in to comment.