From ba84d899c22db89df6e34790407e84281a5f6612 Mon Sep 17 00:00:00 2001 From: Thomas Steur Date: Wed, 30 Oct 2013 14:42:44 +1300 Subject: [PATCH] refs #4044 this could fix the filter_limit and filter_offset is applied twice. Once in Live.getLastVisitDetails and once in ResponseBuilder. Unfortunately, cannot run all tests on my local machine to make sure everything still works. --- plugins/Live/API.php | 517 ++++++--------------------------------- plugins/Live/Visitor.php | 387 +++++++++++++++++++++++++++++ 2 files changed, 458 insertions(+), 446 deletions(-) diff --git a/plugins/Live/API.php b/plugins/Live/API.php index 52e6020f7dc..3aaf5993b0d 100644 --- a/plugins/Live/API.php +++ b/plugins/Live/API.php @@ -13,8 +13,6 @@ use Exception; use Piwik\Common; use Piwik\Config; -use Piwik\DataAccess\LogAggregator; -use Piwik\DataTable\Filter\ColumnDelete; use Piwik\DataTable\Row; use Piwik\DataTable; use Piwik\Date; @@ -27,8 +25,6 @@ use Piwik\Plugins\SitesManager\API as APISitesManager; use Piwik\Segment; use Piwik\Site; -use Piwik\Tracker\Action; -use Piwik\Tracker\GoalManager; use Piwik\Tracker; /** @@ -121,8 +117,12 @@ public function getCounters($idSite, $lastMinutes, $segment = false) public function getLastVisitsForVisitor($visitorId, $idSite, $filter_limit = 10, $flat = false) { Piwik::checkUserHasViewAccess($idSite); - $visitorDetails = $this->loadLastVisitorDetailsFromDatabase($idSite, $period = false, $date = false, $segment = false, $filter_limit, $filter_offset = false, $visitorId); - $table = $this->getCleanedVisitorsFromDetails($visitorDetails, $idSite, $flat); + + $numLastVisitorsToFetch = $filter_limit; + + $table = $this->loadLastVisitorDetailsFromDatabase($idSite, $period = false, $date = false, $segment = false, $numLastVisitorsToFetch, $visitorId); + $this->addFilterToCleanVisitors($table, $idSite, $flat); + return $table; } @@ -134,22 +134,25 @@ public function getLastVisitsForVisitor($visitorId, $idSite, $filter_limit = 10, * @param bool|string $period Period to restrict to when looking at the logs * @param bool|string $date Date to restrict to * @param bool|int $segment (optional) Number of visits rows to return - * @param bool|int $filter_limit (optional) Only return X visits - * @param bool|int $filter_offset (optional) Skip the first X visits (useful when paginating) + * @param bool|int $numLastVisitorsToFetch (optional) Only return the last X visits. By default the last GET['filter_offset']+GET['filter_limit'] are returned. * @param bool|int $minTimestamp (optional) Minimum timestamp to restrict the query to (useful when paginating or refreshing visits) * @param bool $flat * @param bool $doNotFetchActions * @return DataTable */ - public function getLastVisitsDetails($idSite, $period = false, $date = false, $segment = false, $filter_limit = false, - $filter_offset = false, $minTimestamp = false, $flat = false, $doNotFetchActions = false) + public function getLastVisitsDetails($idSite, $period = false, $date = false, $segment = false, $numLastVisitorsToFetch = false, $minTimestamp = false, $flat = false, $doNotFetchActions = false) { - if (empty($filter_limit)) { - $filter_limit = 10; + if (false === $numLastVisitorsToFetch) { + $filter_limit = Common::getRequestVar('filter_limit', 10, 'int'); + $filter_offset = Common::getRequestVar('filter_offset', 0, 'int'); + + $numLastVisitorsToFetch = $filter_limit + $filter_offset; } + Piwik::checkUserHasViewAccess($idSite); - $visitorDetails = $this->loadLastVisitorDetailsFromDatabase($idSite, $period, $date, $segment, $filter_limit, $filter_offset, $visitorId = false, $minTimestamp); - $dataTable = $this->getCleanedVisitorsFromDetails($visitorDetails, $idSite, $flat, $doNotFetchActions); + $dataTable = $this->loadLastVisitorDetailsFromDatabase($idSite, $period, $date, $segment, $numLastVisitorsToFetch, $visitorId = false, $minTimestamp); + $this->addFilterToCleanVisitors($dataTable, $idSite, $flat, $doNotFetchActions); + return $dataTable; } @@ -157,8 +160,8 @@ public function getLastVisitsDetails($idSite, $period = false, $date = false, $s * Returns an array describing a visitor using her last visits (uses a maximum of 100). * * @param int $idSite Site ID - * @param string|false $visitorId The ID of the visitor whose profile to retrieve. - * @param string|false $segment + * @param bool|false|string $visitorId The ID of the visitor whose profile to retrieve. + * @param bool|false|string $segment * @param bool $checkForLatLong If true, hasLatLong will appear in the output and be true if * one of the first 100 visits has a latitude/longitude. * @return array @@ -172,8 +175,8 @@ public function getVisitorProfile($idSite, $visitorId = false, $segment = false, $newSegment = ($segment === false ? '' : $segment . ';') . 'visitorId==' . $visitorId; $visits = $this->getLastVisitsDetails($idSite, $period = false, $date = false, $newSegment, - $filter_limit = self::VISITOR_PROFILE_MAX_VISITS_TO_AGGREGATE, - $filter_offset = false, $overrideVisitorId = false, + $numVisitorsToFetch = self::VISITOR_PROFILE_MAX_VISITS_TO_AGGREGATE, + $overrideVisitorId = false, $minTimestamp = false); if ($visits->getRowsCount() == 0) { return array(); @@ -370,23 +373,25 @@ public function getVisitorProfile($idSite, $visitorId = false, $segment = false, * Returns the visitor ID of the most recent visit. * * @param int $idSite - * @param string|false $segment + * @param bool|string $segment * @return string */ public function getMostRecentVisitorId($idSite, $segment = false) { Piwik::checkUserHasViewAccess($idSite); - $visitDetails = $this->loadLastVisitorDetailsFromDatabase( - $idSite, $period = false, $date = false, $segment, $filter_limit = 1, $filter_offset = false, + $dataTable = $this->loadLastVisitorDetailsFromDatabase( + $idSite, $period = false, $date = false, $segment, $numVisitorsToFetch = 1, $visitorId = false, $minTimestamp = false ); - if (empty($visitDetails)) { + if (0 >= $dataTable->getRowsCount()) { return false; } - $visitor = new Visitor($visitDetails[0]); + $visitDetails = $dataTable->getFirstRow()->getColumns(); + $visitor = new Visitor($visitDetails); + return $visitor->getVisitorId(); } @@ -494,177 +499,62 @@ public static function getReferrerSummaryForVisit($visit) */ public function getLastVisits($idSite, $filter_limit = 10, $minTimestamp = false) { - return $this->getLastVisitsDetails($idSite, $period = false, $date = false, $segment = false, $filter_limit, $filter_offset = false, $minTimestamp, $flat = false); + return $this->getLastVisitsDetails($idSite, $period = false, $date = false, $segment = false, $numLastVisitorsToFetch = $filter_limit, $minTimestamp, $flat = false); } /** * For an array of visits, query the list of pages for this visit * as well as make the data human readable - * @param array $visitorDetails + * @param DataTable $dataTable * @param int $idSite * @param bool $flat whether to flatten the array (eg. 'customVariables' names/values will appear in the root array rather than in 'customVariables' key * @param bool $doNotFetchActions If set to true, we only fetch visit info and not actions (much faster) - * - * @return DataTable - */ - private function getCleanedVisitorsFromDetails($visitorDetails, $idSite, $flat = false, $doNotFetchActions = false) - { - $actionsLimit = (int)Config::getInstance()->General['visitor_log_maximum_actions_per_visit']; - - $table = new DataTable(); - - $site = new Site($idSite); - $timezone = $site->getTimezone(); - $currencies = APISitesManager::getInstance()->getCurrencySymbols(); - foreach ($visitorDetails as $visitorDetail) { - $this->cleanVisitorDetails($visitorDetail, $idSite); - $visitor = new Visitor($visitorDetail); - $visitorDetailsArray = $visitor->getAllVisitorDetails(); - - $visitorDetailsArray['siteCurrency'] = $site->getCurrency(); - $visitorDetailsArray['siteCurrencySymbol'] = @$currencies[$site->getCurrency()]; - $visitorDetailsArray['serverTimestamp'] = $visitorDetailsArray['lastActionTimestamp']; - $dateTimeVisit = Date::factory($visitorDetailsArray['lastActionTimestamp'], $timezone); - $visitorDetailsArray['serverTimePretty'] = $dateTimeVisit->getLocalized('%time%'); - $visitorDetailsArray['serverDatePretty'] = $dateTimeVisit->getLocalized(Piwik::translate('CoreHome_ShortDateFormat')); - - $dateTimeVisitFirstAction = Date::factory($visitorDetailsArray['firstActionTimestamp'], $timezone); - $visitorDetailsArray['serverDatePrettyFirstAction'] = $dateTimeVisitFirstAction->getLocalized(Piwik::translate('CoreHome_ShortDateFormat')); - $visitorDetailsArray['serverTimePrettyFirstAction'] = $dateTimeVisitFirstAction->getLocalized('%time%'); - - $visitorDetailsArray['actionDetails'] = array(); - if (!$doNotFetchActions) { - $visitorDetailsArray = $this->enrichVisitorArrayWithActions($visitorDetailsArray, $actionsLimit, $timezone); - } - - if ($flat) { - $visitorDetailsArray = $this->flattenVisitorDetailsArray($visitorDetailsArray); - } - $table->addRowFromArray(array(Row::COLUMNS => $visitorDetailsArray)); - } - return $table; - } - - private function getCustomVariablePrettyKey($key) - { - $rename = array( - Tracker\ActionSiteSearch::CVAR_KEY_SEARCH_CATEGORY => Piwik::translate('Actions_ColumnSearchCategory'), - Tracker\ActionSiteSearch::CVAR_KEY_SEARCH_COUNT => Piwik::translate('Actions_ColumnSearchResultsCount'), - ); - if (isset($rename[$key])) { - return $rename[$key]; - } - return $key; - } - - /** - * The &flat=1 feature is used by API.getSuggestedValuesForSegment - * - * @param $visitorDetailsArray - * @return array */ - private function flattenVisitorDetailsArray($visitorDetailsArray) + private function addFilterToCleanVisitors(DataTable $dataTable, $idSite, $flat = false, $doNotFetchActions = false) { - // NOTE: if you flatten more fields from the "actionDetails" array - // ==> also update API/API.php getSuggestedValuesForSegment(), the $segmentsNeedActionsInfo array - - // flatten visit custom variables - if (is_array($visitorDetailsArray['customVariables'])) { - foreach ($visitorDetailsArray['customVariables'] as $thisCustomVar) { - $visitorDetailsArray = array_merge($visitorDetailsArray, $thisCustomVar); - } - unset($visitorDetailsArray['customVariables']); - } - - // flatten page views custom variables - $count = 1; - foreach ($visitorDetailsArray['actionDetails'] as $action) { - if (!empty($action['customVariables'])) { - foreach ($action['customVariables'] as $thisCustomVar) { - foreach ($thisCustomVar as $cvKey => $cvValue) { - $flattenedKeyName = $cvKey . ColumnDelete::APPEND_TO_COLUMN_NAME_TO_KEEP . $count; - $visitorDetailsArray[$flattenedKeyName] = $cvValue; - $count++; - } + $dataTable->queueFilter(function ($table) use ($idSite, $flat, $doNotFetchActions) { + /** @var DataTable $table */ + $actionsLimit = (int)Config::getInstance()->General['visitor_log_maximum_actions_per_visit']; + + $site = new Site($idSite); + $timezone = $site->getTimezone(); + $currencies = APISitesManager::getInstance()->getCurrencySymbols(); + + foreach ($table->getRows() as $visitorDetailRow) { + $visitorDetailsArray = Visitor::cleanVisitorDetails($visitorDetailRow->getColumns()); + + $visitor = new Visitor($visitorDetailsArray); + $visitorDetailsArray = $visitor->getAllVisitorDetails(); + + $visitorDetailsArray['siteCurrency'] = $site->getCurrency(); + $visitorDetailsArray['siteCurrencySymbol'] = @$currencies[$site->getCurrency()]; + $visitorDetailsArray['serverTimestamp'] = $visitorDetailsArray['lastActionTimestamp']; + $dateTimeVisit = Date::factory($visitorDetailsArray['lastActionTimestamp'], $timezone); + $visitorDetailsArray['serverTimePretty'] = $dateTimeVisit->getLocalized('%time%'); + $visitorDetailsArray['serverDatePretty'] = $dateTimeVisit->getLocalized(Piwik::translate('CoreHome_ShortDateFormat')); + + $dateTimeVisitFirstAction = Date::factory($visitorDetailsArray['firstActionTimestamp'], $timezone); + $visitorDetailsArray['serverDatePrettyFirstAction'] = $dateTimeVisitFirstAction->getLocalized(Piwik::translate('CoreHome_ShortDateFormat')); + $visitorDetailsArray['serverTimePrettyFirstAction'] = $dateTimeVisitFirstAction->getLocalized('%time%'); + + $visitorDetailsArray['actionDetails'] = array(); + if (!$doNotFetchActions) { + $visitorDetailsArray = Visitor::enrichVisitorArrayWithActions($visitorDetailsArray, $actionsLimit, $timezone); } - } - } - // Flatten Goals - $count = 1; - foreach ($visitorDetailsArray['actionDetails'] as $action) { - if (!empty($action['goalId'])) { - $flattenedKeyName = 'visitConvertedGoalId' . ColumnDelete::APPEND_TO_COLUMN_NAME_TO_KEEP . $count; - $visitorDetailsArray[$flattenedKeyName] = $action['goalId']; - $count++; - } - } - - // Flatten Page Titles/URLs - $count = 1; - foreach ($visitorDetailsArray['actionDetails'] as $action) { - if (!empty($action['url'])) { - $flattenedKeyName = 'pageUrl' . ColumnDelete::APPEND_TO_COLUMN_NAME_TO_KEEP . $count; - $visitorDetailsArray[$flattenedKeyName] = $action['url']; - } - - if (!empty($action['pageTitle'])) { - $flattenedKeyName = 'pageTitle' . ColumnDelete::APPEND_TO_COLUMN_NAME_TO_KEEP . $count; - $visitorDetailsArray[$flattenedKeyName] = $action['pageTitle']; - } - - if (!empty($action['siteSearchKeyword'])) { - $flattenedKeyName = 'siteSearchKeyword' . ColumnDelete::APPEND_TO_COLUMN_NAME_TO_KEEP . $count; - $visitorDetailsArray[$flattenedKeyName] = $action['siteSearchKeyword']; - } - $count++; - } - - // Entry/exit pages - $firstAction = $lastAction = false; - foreach ($visitorDetailsArray['actionDetails'] as $action) { - if ($action['type'] == 'action') { - if (empty($firstAction)) { - $firstAction = $action; + if ($flat) { + $visitorDetailsArray = Visitor::flattenVisitorDetailsArray($visitorDetailsArray); } - $lastAction = $action; - } - } - if (!empty($firstAction['pageTitle'])) { - $visitorDetailsArray['entryPageTitle'] = $firstAction['pageTitle']; - } - if (!empty($firstAction['url'])) { - $visitorDetailsArray['entryPageUrl'] = $firstAction['url']; - } - if (!empty($lastAction['pageTitle'])) { - $visitorDetailsArray['exitPageTitle'] = $lastAction['pageTitle']; - } - if (!empty($lastAction['url'])) { - $visitorDetailsArray['exitPageUrl'] = $lastAction['url']; - } + $visitorDetailRow->setColumns($visitorDetailsArray); + } - return $visitorDetailsArray; + return $table; + }); } - private function sortByServerTime($a, $b) + private function loadLastVisitorDetailsFromDatabase($idSite, $period, $date, $segment = false, $numLastVisitorsToFetch = 100, $visitorId = false, $minTimestamp = false) { - $ta = strtotime($a['serverTimePretty']); - $tb = strtotime($b['serverTimePretty']); - return $ta < $tb - ? -1 - : ($ta == $tb - ? 0 - : 1); - } - - private function loadLastVisitorDetailsFromDatabase($idSite, $period, $date, $segment = false, $filter_limit = false, - $filter_offset = false, $visitorId = false, $minTimestamp = false) - { - if (empty($filter_limit)) { - $filter_limit = 100; - } - $where = $whereBind = array(); $where[] = "log_visit.idsite = ? "; $whereBind[] = $idSite; @@ -682,8 +572,7 @@ private function loadLastVisitorDetailsFromDatabase($idSite, $period, $date, $se // If no other filter, only look at the last 24 hours of stats if (empty($visitorId) - && empty($filter_limit) - && empty($filter_offset) + && empty($numLastVisitorsToFetch) && empty($period) && empty($date) ) { @@ -742,7 +631,7 @@ private function loadLastVisitorDetailsFromDatabase($idSite, $period, $date, $se $from = "log_visit"; $subQuery = $segment->getSelectQuery($select, $from, $where, $whereBind, $orderBy); - $sqlLimit = $filter_limit >= 1 ? " LIMIT " . (int)$filter_offset . ", " . (int)$filter_limit : ""; + $sqlLimit = $numLastVisitorsToFetch >= 1 ? " LIMIT 0, " . (int)$numLastVisitorsToFetch : ""; // Group by idvisit so that a visitor converting 2 goals only appears once $sql = " @@ -761,273 +650,9 @@ private function loadLastVisitorDetailsFromDatabase($idSite, $period, $date, $se exit; } - return $data; - } + $dataTable = new DataTable(); + $dataTable->addRowsFromSimpleArray($data); - /** - * Removes fields that are not meant to be displayed (md5 config hash) - * Or that the user should only access if he is super user or admin (cookie, IP) - * - * @param array $visitorDetails - * @param int $idSite - * @return void - */ - private function cleanVisitorDetails(&$visitorDetails, $idSite) - { - $toUnset = array('config_id'); - if (Piwik::isUserIsAnonymous()) { - $toUnset[] = 'idvisitor'; - $toUnset[] = 'location_ip'; - } - foreach ($toUnset as $keyName) { - if (isset($visitorDetails[$keyName])) { - unset($visitorDetails[$keyName]); - } - } - } - - /** - * @param $visitorDetailsArray - * @param $actionsLimit - * @param $timezone - * @return array - */ - private function enrichVisitorArrayWithActions($visitorDetailsArray, $actionsLimit, $timezone) - { - $idVisit = $visitorDetailsArray['idVisit']; - - $sqlCustomVariables = ''; - for ($i = 1; $i <= Tracker::MAX_CUSTOM_VARIABLES; $i++) { - $sqlCustomVariables .= ', custom_var_k' . $i . ', custom_var_v' . $i; - } - // The second join is a LEFT join to allow returning records that don't have a matching page title - // eg. Downloads, Outlinks. For these, idaction_name is set to 0 - $sql = " - SELECT - COALESCE(log_action_event_category.type, log_action.type, log_action_title.type) AS type, - log_action.name AS url, - log_action.url_prefix, - log_action_title.name AS pageTitle, - log_action.idaction AS pageIdAction, - log_link_visit_action.server_time as serverTimePretty, - log_link_visit_action.time_spent_ref_action as timeSpentRef, - log_link_visit_action.idlink_va AS pageId, - log_link_visit_action.custom_float - ". $sqlCustomVariables . ", - log_action_event_category.name AS eventCategory, - log_action_event_action.name as eventAction - FROM " . Common::prefixTable('log_link_visit_action') . " AS log_link_visit_action - LEFT JOIN " . Common::prefixTable('log_action') . " AS log_action - ON log_link_visit_action.idaction_url = log_action.idaction - LEFT JOIN " . Common::prefixTable('log_action') . " AS log_action_title - ON log_link_visit_action.idaction_name = log_action_title.idaction - LEFT JOIN " . Common::prefixTable('log_action') . " AS log_action_event_category - ON log_link_visit_action.idaction_event_category = log_action_event_category.idaction - LEFT JOIN " . Common::prefixTable('log_action') . " AS log_action_event_action - ON log_link_visit_action.idaction_event_action = log_action_event_action.idaction - WHERE log_link_visit_action.idvisit = ? - ORDER BY server_time ASC - LIMIT 0, $actionsLimit - "; - $actionDetails = Db::fetchAll($sql, array($idVisit)); - - foreach ($actionDetails as $actionIdx => &$actionDetail) { - $actionDetail =& $actionDetails[$actionIdx]; - $customVariablesPage = array(); - for ($i = 1; $i <= Tracker::MAX_CUSTOM_VARIABLES; $i++) { - if (!empty($actionDetail['custom_var_k' . $i])) { - $cvarKey = $actionDetail['custom_var_k' . $i]; - $cvarKey = $this->getCustomVariablePrettyKey($cvarKey); - $customVariablesPage[$i] = array( - 'customVariablePageName' . $i => $cvarKey, - 'customVariablePageValue' . $i => $actionDetail['custom_var_v' . $i], - ); - } - unset($actionDetail['custom_var_k' . $i]); - unset($actionDetail['custom_var_v' . $i]); - } - if (!empty($customVariablesPage)) { - $actionDetail['customVariables'] = $customVariablesPage; - } - - - if($actionDetail['type'] == Action::TYPE_EVENT_CATEGORY) { - // Handle Event - if(strlen($actionDetail['pageTitle']) > 0) { - $actionDetail['eventName'] = $actionDetail['pageTitle']; - } - - unset($actionDetail['pageTitle']); - - } else if ($actionDetail['type'] == Action::TYPE_SITE_SEARCH) { - // Handle Site Search - $actionDetail['siteSearchKeyword'] = $actionDetail['pageTitle']; - unset($actionDetail['pageTitle']); - } - - // Event value / Generation time - if($actionDetail['type'] == Action::TYPE_EVENT_CATEGORY) { - if(strlen($actionDetail['custom_float']) > 0) { - $actionDetail['eventValue'] = $actionDetail['custom_float']; - } - } elseif ($actionDetail['custom_float'] > 0) { - $actionDetail['generationTime'] = \Piwik\MetricsFormatter::getPrettyTimeFromSeconds($actionDetail['custom_float'] / 1000); - } - unset($actionDetail['custom_float']); - - if($actionDetail['type'] != Action::TYPE_EVENT_CATEGORY) { - unset($actionDetail['eventCategory']); - unset($actionDetail['eventAction']); - } - - // Reconstruct url from prefix - $actionDetail['url'] = Tracker\PageUrl::reconstructNormalizedUrl($actionDetail['url'], $actionDetail['url_prefix']); - unset($actionDetail['url_prefix']); - - // Set the time spent for this action (which is the timeSpentRef of the next action) - if (isset($actionDetails[$actionIdx + 1])) { - $actionDetail['timeSpent'] = $actionDetails[$actionIdx + 1]['timeSpentRef']; - $actionDetail['timeSpentPretty'] = \Piwik\MetricsFormatter::getPrettyTimeFromSeconds($actionDetail['timeSpent']); - } - unset($actionDetails[$actionIdx]['timeSpentRef']); // not needed after timeSpent is added - - } - - // If the visitor converted a goal, we shall select all Goals - $sql = " - SELECT - 'goal' as type, - goal.name as goalName, - goal.idgoal as goalId, - goal.revenue as revenue, - log_conversion.idlink_va as goalPageId, - log_conversion.server_time as serverTimePretty, - log_conversion.url as url - FROM " . Common::prefixTable('log_conversion') . " AS log_conversion - LEFT JOIN " . Common::prefixTable('goal') . " AS goal - ON (goal.idsite = log_conversion.idsite - AND - goal.idgoal = log_conversion.idgoal) - AND goal.deleted = 0 - WHERE log_conversion.idvisit = ? - AND log_conversion.idgoal > 0 - ORDER BY server_time ASC - LIMIT 0, $actionsLimit - "; - $goalDetails = Db::fetchAll($sql, array($idVisit)); - - $sql = "SELECT - case idgoal when " . GoalManager::IDGOAL_CART . " then '" . Piwik::LABEL_ID_GOAL_IS_ECOMMERCE_CART . "' else '" . Piwik::LABEL_ID_GOAL_IS_ECOMMERCE_ORDER . "' end as type, - idorder as orderId, - " . LogAggregator::getSqlRevenue('revenue') . " as revenue, - " . LogAggregator::getSqlRevenue('revenue_subtotal') . " as revenueSubTotal, - " . LogAggregator::getSqlRevenue('revenue_tax') . " as revenueTax, - " . LogAggregator::getSqlRevenue('revenue_shipping') . " as revenueShipping, - " . LogAggregator::getSqlRevenue('revenue_discount') . " as revenueDiscount, - items as items, - - log_conversion.server_time as serverTimePretty - FROM " . Common::prefixTable('log_conversion') . " AS log_conversion - WHERE idvisit = ? - AND idgoal <= " . GoalManager::IDGOAL_ORDER . " - ORDER BY server_time ASC - LIMIT 0, $actionsLimit"; - $ecommerceDetails = Db::fetchAll($sql, array($idVisit)); - - foreach ($ecommerceDetails as &$ecommerceDetail) { - if ($ecommerceDetail['type'] == Piwik::LABEL_ID_GOAL_IS_ECOMMERCE_CART) { - unset($ecommerceDetail['orderId']); - unset($ecommerceDetail['revenueSubTotal']); - unset($ecommerceDetail['revenueTax']); - unset($ecommerceDetail['revenueShipping']); - unset($ecommerceDetail['revenueDiscount']); - } - - // 25.00 => 25 - foreach ($ecommerceDetail as $column => $value) { - if (strpos($column, 'revenue') !== false) { - if ($value == round($value)) { - $ecommerceDetail[$column] = round($value); - } - } - } - } - - // Enrich ecommerce carts/orders with the list of products - usort($ecommerceDetails, array($this, 'sortByServerTime')); - foreach ($ecommerceDetails as $key => &$ecommerceConversion) { - $sql = "SELECT - log_action_sku.name as itemSKU, - log_action_name.name as itemName, - log_action_category.name as itemCategory, - " . LogAggregator::getSqlRevenue('price') . " as price, - quantity as quantity - FROM " . Common::prefixTable('log_conversion_item') . " - INNER JOIN " . Common::prefixTable('log_action') . " AS log_action_sku - ON idaction_sku = log_action_sku.idaction - LEFT JOIN " . Common::prefixTable('log_action') . " AS log_action_name - ON idaction_name = log_action_name.idaction - LEFT JOIN " . Common::prefixTable('log_action') . " AS log_action_category - ON idaction_category = log_action_category.idaction - WHERE idvisit = ? - AND idorder = ? - AND deleted = 0 - LIMIT 0, $actionsLimit - "; - $bind = array($idVisit, isset($ecommerceConversion['orderId']) - ? $ecommerceConversion['orderId'] - : GoalManager::ITEM_IDORDER_ABANDONED_CART - ); - - $itemsDetails = Db::fetchAll($sql, $bind); - foreach ($itemsDetails as &$detail) { - if ($detail['price'] == round($detail['price'])) { - $detail['price'] = round($detail['price']); - } - } - $ecommerceConversion['itemDetails'] = $itemsDetails; - } - - $actions = array_merge($actionDetails, $goalDetails, $ecommerceDetails); - - usort($actions, array($this, 'sortByServerTime')); - - $visitorDetailsArray['actionDetails'] = $actions; - foreach ($visitorDetailsArray['actionDetails'] as &$details) { - switch ($details['type']) { - case 'goal': - $details['icon'] = 'plugins/Zeitgeist/images/goal.png'; - break; - case Piwik::LABEL_ID_GOAL_IS_ECOMMERCE_ORDER: - case Piwik::LABEL_ID_GOAL_IS_ECOMMERCE_CART: - $details['icon'] = 'plugins/Zeitgeist/images/' . $details['type'] . '.gif'; - break; - case Action::TYPE_DOWNLOAD: - $details['type'] = 'download'; - $details['icon'] = 'plugins/Zeitgeist/images/download.png'; - break; - case Action::TYPE_OUTLINK: - $details['type'] = 'outlink'; - $details['icon'] = 'plugins/Zeitgeist/images/link.gif'; - break; - case Action::TYPE_SITE_SEARCH: - $details['type'] = 'search'; - $details['icon'] = 'plugins/Zeitgeist/images/search_ico.png'; - break; - case Action::TYPE_EVENT_CATEGORY: - $details['type'] = 'event'; - $details['icon'] = 'plugins/Zeitgeist/images/event.png'; - break; - default: - $details['type'] = 'action'; - $details['icon'] = null; - break; - } - // Convert datetimes to the site timezone - $dateTimeVisit = Date::factory($details['serverTimePretty'], $timezone); - $details['serverTimePretty'] = $dateTimeVisit->getLocalized(Piwik::translate('CoreHome_ShortDateFormat') . ' %time%'); - } - $visitorDetailsArray['goalConversions'] = count($goalDetails); - return $visitorDetailsArray; + return $dataTable; } } diff --git a/plugins/Live/Visitor.php b/plugins/Live/Visitor.php index 47e32c879b9..34643a0cfda 100644 --- a/plugins/Live/Visitor.php +++ b/plugins/Live/Visitor.php @@ -19,6 +19,12 @@ use Piwik\Tracker; use Piwik\Tracker\Visit; use Piwik\UrlHelper; +use Piwik\Date; +use Piwik\Db; +use Piwik\Tracker\Action; +use Piwik\Tracker\GoalManager; +use Piwik\DataAccess\LogAggregator; +use Piwik\DataTable\Filter\ColumnDelete; /** * @see plugins/Referrers/functions.php @@ -594,4 +600,385 @@ function isVisitorGoalConverted() { return $this->details['visit_goal_converted']; } + + /** + * Removes fields that are not meant to be displayed (md5 config hash) + * Or that the user should only access if he is super user or admin (cookie, IP) + * + * @param array $visitorDetails + * @return array + */ + public static function cleanVisitorDetails($visitorDetails) + { + $toUnset = array('config_id'); + if (Piwik::isUserIsAnonymous()) { + $toUnset[] = 'idvisitor'; + $toUnset[] = 'location_ip'; + } + foreach ($toUnset as $keyName) { + if (isset($visitorDetails[$keyName])) { + unset($visitorDetails[$keyName]); + } + } + + return $visitorDetails; + } + + /** + * The &flat=1 feature is used by API.getSuggestedValuesForSegment + * + * @param $visitorDetailsArray + * @return array + */ + public static function flattenVisitorDetailsArray($visitorDetailsArray) + { + // NOTE: if you flatten more fields from the "actionDetails" array + // ==> also update API/API.php getSuggestedValuesForSegment(), the $segmentsNeedActionsInfo array + + // flatten visit custom variables + if (is_array($visitorDetailsArray['customVariables'])) { + foreach ($visitorDetailsArray['customVariables'] as $thisCustomVar) { + $visitorDetailsArray = array_merge($visitorDetailsArray, $thisCustomVar); + } + unset($visitorDetailsArray['customVariables']); + } + + // flatten page views custom variables + $count = 1; + foreach ($visitorDetailsArray['actionDetails'] as $action) { + if (!empty($action['customVariables'])) { + foreach ($action['customVariables'] as $thisCustomVar) { + foreach ($thisCustomVar as $cvKey => $cvValue) { + $flattenedKeyName = $cvKey . ColumnDelete::APPEND_TO_COLUMN_NAME_TO_KEEP . $count; + $visitorDetailsArray[$flattenedKeyName] = $cvValue; + $count++; + } + } + } + } + + // Flatten Goals + $count = 1; + foreach ($visitorDetailsArray['actionDetails'] as $action) { + if (!empty($action['goalId'])) { + $flattenedKeyName = 'visitConvertedGoalId' . ColumnDelete::APPEND_TO_COLUMN_NAME_TO_KEEP . $count; + $visitorDetailsArray[$flattenedKeyName] = $action['goalId']; + $count++; + } + } + + // Flatten Page Titles/URLs + $count = 1; + foreach ($visitorDetailsArray['actionDetails'] as $action) { + if (!empty($action['url'])) { + $flattenedKeyName = 'pageUrl' . ColumnDelete::APPEND_TO_COLUMN_NAME_TO_KEEP . $count; + $visitorDetailsArray[$flattenedKeyName] = $action['url']; + } + + if (!empty($action['pageTitle'])) { + $flattenedKeyName = 'pageTitle' . ColumnDelete::APPEND_TO_COLUMN_NAME_TO_KEEP . $count; + $visitorDetailsArray[$flattenedKeyName] = $action['pageTitle']; + } + + if (!empty($action['siteSearchKeyword'])) { + $flattenedKeyName = 'siteSearchKeyword' . ColumnDelete::APPEND_TO_COLUMN_NAME_TO_KEEP . $count; + $visitorDetailsArray[$flattenedKeyName] = $action['siteSearchKeyword']; + } + $count++; + } + + // Entry/exit pages + $firstAction = $lastAction = false; + foreach ($visitorDetailsArray['actionDetails'] as $action) { + if ($action['type'] == 'action') { + if (empty($firstAction)) { + $firstAction = $action; + } + $lastAction = $action; + } + } + + if (!empty($firstAction['pageTitle'])) { + $visitorDetailsArray['entryPageTitle'] = $firstAction['pageTitle']; + } + if (!empty($firstAction['url'])) { + $visitorDetailsArray['entryPageUrl'] = $firstAction['url']; + } + if (!empty($lastAction['pageTitle'])) { + $visitorDetailsArray['exitPageTitle'] = $lastAction['pageTitle']; + } + if (!empty($lastAction['url'])) { + $visitorDetailsArray['exitPageUrl'] = $lastAction['url']; + } + + return $visitorDetailsArray; + } + + /** + * @param $visitorDetailsArray + * @param $actionsLimit + * @param $timezone + * @return array + */ + public static function enrichVisitorArrayWithActions($visitorDetailsArray, $actionsLimit, $timezone) + { + $idVisit = $visitorDetailsArray['idVisit']; + + $sqlCustomVariables = ''; + for ($i = 1; $i <= Tracker::MAX_CUSTOM_VARIABLES; $i++) { + $sqlCustomVariables .= ', custom_var_k' . $i . ', custom_var_v' . $i; + } + // The second join is a LEFT join to allow returning records that don't have a matching page title + // eg. Downloads, Outlinks. For these, idaction_name is set to 0 + $sql = " + SELECT + COALESCE(log_action_event_category.type, log_action.type, log_action_title.type) AS type, + log_action.name AS url, + log_action.url_prefix, + log_action_title.name AS pageTitle, + log_action.idaction AS pageIdAction, + log_link_visit_action.server_time as serverTimePretty, + log_link_visit_action.time_spent_ref_action as timeSpentRef, + log_link_visit_action.idlink_va AS pageId, + log_link_visit_action.custom_float + ". $sqlCustomVariables . ", + log_action_event_category.name AS eventCategory, + log_action_event_action.name as eventAction + FROM " . Common::prefixTable('log_link_visit_action') . " AS log_link_visit_action + LEFT JOIN " . Common::prefixTable('log_action') . " AS log_action + ON log_link_visit_action.idaction_url = log_action.idaction + LEFT JOIN " . Common::prefixTable('log_action') . " AS log_action_title + ON log_link_visit_action.idaction_name = log_action_title.idaction + LEFT JOIN " . Common::prefixTable('log_action') . " AS log_action_event_category + ON log_link_visit_action.idaction_event_category = log_action_event_category.idaction + LEFT JOIN " . Common::prefixTable('log_action') . " AS log_action_event_action + ON log_link_visit_action.idaction_event_action = log_action_event_action.idaction + WHERE log_link_visit_action.idvisit = ? + ORDER BY server_time ASC + LIMIT 0, $actionsLimit + "; + $actionDetails = Db::fetchAll($sql, array($idVisit)); + + foreach ($actionDetails as $actionIdx => &$actionDetail) { + $actionDetail =& $actionDetails[$actionIdx]; + $customVariablesPage = array(); + for ($i = 1; $i <= Tracker::MAX_CUSTOM_VARIABLES; $i++) { + if (!empty($actionDetail['custom_var_k' . $i])) { + $cvarKey = $actionDetail['custom_var_k' . $i]; + $cvarKey = static::getCustomVariablePrettyKey($cvarKey); + $customVariablesPage[$i] = array( + 'customVariablePageName' . $i => $cvarKey, + 'customVariablePageValue' . $i => $actionDetail['custom_var_v' . $i], + ); + } + unset($actionDetail['custom_var_k' . $i]); + unset($actionDetail['custom_var_v' . $i]); + } + if (!empty($customVariablesPage)) { + $actionDetail['customVariables'] = $customVariablesPage; + } + + + if($actionDetail['type'] == Action::TYPE_EVENT_CATEGORY) { + // Handle Event + if(strlen($actionDetail['pageTitle']) > 0) { + $actionDetail['eventName'] = $actionDetail['pageTitle']; + } + + unset($actionDetail['pageTitle']); + + } else if ($actionDetail['type'] == Action::TYPE_SITE_SEARCH) { + // Handle Site Search + $actionDetail['siteSearchKeyword'] = $actionDetail['pageTitle']; + unset($actionDetail['pageTitle']); + } + + // Event value / Generation time + if($actionDetail['type'] == Action::TYPE_EVENT_CATEGORY) { + if(strlen($actionDetail['custom_float']) > 0) { + $actionDetail['eventValue'] = $actionDetail['custom_float']; + } + } elseif ($actionDetail['custom_float'] > 0) { + $actionDetail['generationTime'] = \Piwik\MetricsFormatter::getPrettyTimeFromSeconds($actionDetail['custom_float'] / 1000); + } + unset($actionDetail['custom_float']); + + if($actionDetail['type'] != Action::TYPE_EVENT_CATEGORY) { + unset($actionDetail['eventCategory']); + unset($actionDetail['eventAction']); + } + + // Reconstruct url from prefix + $actionDetail['url'] = Tracker\PageUrl::reconstructNormalizedUrl($actionDetail['url'], $actionDetail['url_prefix']); + unset($actionDetail['url_prefix']); + + // Set the time spent for this action (which is the timeSpentRef of the next action) + if (isset($actionDetails[$actionIdx + 1])) { + $actionDetail['timeSpent'] = $actionDetails[$actionIdx + 1]['timeSpentRef']; + $actionDetail['timeSpentPretty'] = \Piwik\MetricsFormatter::getPrettyTimeFromSeconds($actionDetail['timeSpent']); + } + unset($actionDetails[$actionIdx]['timeSpentRef']); // not needed after timeSpent is added + + } + + // If the visitor converted a goal, we shall select all Goals + $sql = " + SELECT + 'goal' as type, + goal.name as goalName, + goal.idgoal as goalId, + goal.revenue as revenue, + log_conversion.idlink_va as goalPageId, + log_conversion.server_time as serverTimePretty, + log_conversion.url as url + FROM " . Common::prefixTable('log_conversion') . " AS log_conversion + LEFT JOIN " . Common::prefixTable('goal') . " AS goal + ON (goal.idsite = log_conversion.idsite + AND + goal.idgoal = log_conversion.idgoal) + AND goal.deleted = 0 + WHERE log_conversion.idvisit = ? + AND log_conversion.idgoal > 0 + ORDER BY server_time ASC + LIMIT 0, $actionsLimit + "; + $goalDetails = Db::fetchAll($sql, array($idVisit)); + + $sql = "SELECT + case idgoal when " . GoalManager::IDGOAL_CART . " then '" . Piwik::LABEL_ID_GOAL_IS_ECOMMERCE_CART . "' else '" . Piwik::LABEL_ID_GOAL_IS_ECOMMERCE_ORDER . "' end as type, + idorder as orderId, + " . LogAggregator::getSqlRevenue('revenue') . " as revenue, + " . LogAggregator::getSqlRevenue('revenue_subtotal') . " as revenueSubTotal, + " . LogAggregator::getSqlRevenue('revenue_tax') . " as revenueTax, + " . LogAggregator::getSqlRevenue('revenue_shipping') . " as revenueShipping, + " . LogAggregator::getSqlRevenue('revenue_discount') . " as revenueDiscount, + items as items, + + log_conversion.server_time as serverTimePretty + FROM " . Common::prefixTable('log_conversion') . " AS log_conversion + WHERE idvisit = ? + AND idgoal <= " . GoalManager::IDGOAL_ORDER . " + ORDER BY server_time ASC + LIMIT 0, $actionsLimit"; + $ecommerceDetails = Db::fetchAll($sql, array($idVisit)); + + foreach ($ecommerceDetails as &$ecommerceDetail) { + if ($ecommerceDetail['type'] == Piwik::LABEL_ID_GOAL_IS_ECOMMERCE_CART) { + unset($ecommerceDetail['orderId']); + unset($ecommerceDetail['revenueSubTotal']); + unset($ecommerceDetail['revenueTax']); + unset($ecommerceDetail['revenueShipping']); + unset($ecommerceDetail['revenueDiscount']); + } + + // 25.00 => 25 + foreach ($ecommerceDetail as $column => $value) { + if (strpos($column, 'revenue') !== false) { + if ($value == round($value)) { + $ecommerceDetail[$column] = round($value); + } + } + } + } + + // Enrich ecommerce carts/orders with the list of products + usort($ecommerceDetails, array('static', 'sortByServerTime')); + foreach ($ecommerceDetails as $key => &$ecommerceConversion) { + $sql = "SELECT + log_action_sku.name as itemSKU, + log_action_name.name as itemName, + log_action_category.name as itemCategory, + " . LogAggregator::getSqlRevenue('price') . " as price, + quantity as quantity + FROM " . Common::prefixTable('log_conversion_item') . " + INNER JOIN " . Common::prefixTable('log_action') . " AS log_action_sku + ON idaction_sku = log_action_sku.idaction + LEFT JOIN " . Common::prefixTable('log_action') . " AS log_action_name + ON idaction_name = log_action_name.idaction + LEFT JOIN " . Common::prefixTable('log_action') . " AS log_action_category + ON idaction_category = log_action_category.idaction + WHERE idvisit = ? + AND idorder = ? + AND deleted = 0 + LIMIT 0, $actionsLimit + "; + $bind = array($idVisit, isset($ecommerceConversion['orderId']) + ? $ecommerceConversion['orderId'] + : GoalManager::ITEM_IDORDER_ABANDONED_CART + ); + + $itemsDetails = Db::fetchAll($sql, $bind); + foreach ($itemsDetails as &$detail) { + if ($detail['price'] == round($detail['price'])) { + $detail['price'] = round($detail['price']); + } + } + $ecommerceConversion['itemDetails'] = $itemsDetails; + } + + $actions = array_merge($actionDetails, $goalDetails, $ecommerceDetails); + + usort($actions, array('static', 'sortByServerTime')); + + $visitorDetailsArray['actionDetails'] = $actions; + foreach ($visitorDetailsArray['actionDetails'] as &$details) { + switch ($details['type']) { + case 'goal': + $details['icon'] = 'plugins/Zeitgeist/images/goal.png'; + break; + case Piwik::LABEL_ID_GOAL_IS_ECOMMERCE_ORDER: + case Piwik::LABEL_ID_GOAL_IS_ECOMMERCE_CART: + $details['icon'] = 'plugins/Zeitgeist/images/' . $details['type'] . '.gif'; + break; + case Action::TYPE_DOWNLOAD: + $details['type'] = 'download'; + $details['icon'] = 'plugins/Zeitgeist/images/download.png'; + break; + case Action::TYPE_OUTLINK: + $details['type'] = 'outlink'; + $details['icon'] = 'plugins/Zeitgeist/images/link.gif'; + break; + case Action::TYPE_SITE_SEARCH: + $details['type'] = 'search'; + $details['icon'] = 'plugins/Zeitgeist/images/search_ico.png'; + break; + case Action::TYPE_EVENT_CATEGORY: + $details['type'] = 'event'; + $details['icon'] = 'plugins/Zeitgeist/images/event.png'; + break; + default: + $details['type'] = 'action'; + $details['icon'] = null; + break; + } + // Convert datetimes to the site timezone + $dateTimeVisit = Date::factory($details['serverTimePretty'], $timezone); + $details['serverTimePretty'] = $dateTimeVisit->getLocalized(Piwik::translate('CoreHome_ShortDateFormat') . ' %time%'); + } + $visitorDetailsArray['goalConversions'] = count($goalDetails); + return $visitorDetailsArray; + } + + private static function getCustomVariablePrettyKey($key) + { + $rename = array( + Tracker\ActionSiteSearch::CVAR_KEY_SEARCH_CATEGORY => Piwik::translate('Actions_ColumnSearchCategory'), + Tracker\ActionSiteSearch::CVAR_KEY_SEARCH_COUNT => Piwik::translate('Actions_ColumnSearchResultsCount'), + ); + if (isset($rename[$key])) { + return $rename[$key]; + } + return $key; + } + + private static function sortByServerTime($a, $b) + { + $ta = strtotime($a['serverTimePretty']); + $tb = strtotime($b['serverTimePretty']); + return $ta < $tb + ? -1 + : ($ta == $tb + ? 0 + : 1); + } }