Refs #2992 Site Search KABOOM, Refs #49

Implementing Site Search tracking & reporting in Piwik core!

 * New Admin UI to customize, for each site, wheter site search is enabled. Also options to set default values to use.
 * New Reports: Searches, Searches with no result, Search categories, Top Pages Following a Search
 * to track "No result keyword" users will have to tag their site with a JS call, or add a new parameter to the search result page &search_count=X (X being zero for no result searches)
 * Reports works with Row evolution, PDF/HTML reports, Piwik Mobile
 * idaction_url is now NULLable because, Site Search records a page with idaction_name == Keyword, and idaction_url == NULL. This ensures that the Site Searches don't create "Page URL Not defined" records. 
 * updates to Tracker JS API, new function trackSiteSearch, also added in PHP tracker
 * New fields in log_visit to track searches
 * new segment, "searches" which can be used to select visitors who did a search ie. searches>0 or those who searched a lot, ie searches>10
 
 TODO: 
  * commmit integration test, TESTING, DOCS, FAQ, release, and a nice Prayer to the universe and the stars, hoping that I can code a major new feature without any bug...
  * It would be awesome to have compatiblity with Transitions so we can see, for a given site search, what are the starting pages and Destination pages
  
 Thank you for your patience Timo, 
 and thank in advance everyone for your help Testing this new feature!
 


git-svn-id: http://dev.piwik.org/svn/trunk@7190 59fd770c-687e-43c8-a1e3-f5a4ff64c105

config/global.ini.php

@@ -369,6 +369,11 @@
 ; maximum length of a Page Title or a Page URL recorded in the log_action.name table
 page_maximum_length = 1024;
 
+; By default, when a request is identified as a "Internal Site Search", the URL will not be recorded. This is for performance reasons
+; (the less unique URLs in Piwik the better). Piwik will track, for each Site Search: "Search Keyword",
+; and optionally the "Search Category" and "Search result count". You can set this to 1 to enable tracking Site Search URLs.
+action_sitesearch_record_url = 0
+
 ; Anonymize a visitor's IP address after testing for "Ip exclude"
 ; This value is the number of octets in IP address to mask; if the AnonymizeIP plugin is deactivated, this value is ignored.
 ; For IPv4 addresses, valid values are 0..4; for IPv6 addresses, valid values are 0..16

core/Archive.php

@@ -75,6 +75,10 @@ abstract class Piwik_Archive
 	const INDEX_ECOMMERCE_ORDERS = 26;
 	const INDEX_ECOMMERCE_ITEM_PRICE_VIEWED = 27;
 
+	// Site Search
+	const INDEX_SITE_SEARCH_HAS_NO_RESULT = 28;
+	const INDEX_PAGE_IS_FOLLOWING_SITE_SEARCH_NB_HITS = 29;
+
 	// Goal reports
 	const INDEX_GOAL_NB_CONVERSIONS = 1;
 	const INDEX_GOAL_REVENUE = 2;
@@ -113,7 +117,8 @@ abstract class Piwik_Archive
 				Piwik_Archive::INDEX_PAGE_ENTRY_NB_ACTIONS => 'entry_nb_actions',
 				Piwik_Archive::INDEX_PAGE_ENTRY_SUM_VISIT_LENGTH => 'entry_sum_visit_length',
 				Piwik_Archive::INDEX_PAGE_ENTRY_BOUNCE_COUNT => 'entry_bounce_count',
-
+				Piwik_Archive::INDEX_PAGE_IS_FOLLOWING_SITE_SEARCH_NB_HITS => 'nb_hits_following_search',
+
 				// Items reports metrics
 				Piwik_Archive::INDEX_ECOMMERCE_ITEM_REVENUE => 'revenue',
 				Piwik_Archive::INDEX_ECOMMERCE_ITEM_QUANTITY => 'quantity',

core/Common.php

@@ -416,7 +416,6 @@ static public function getArrayFromQueryString( $urlQuery )
 		{
 			$urlQuery = substr($urlQuery, 1);
 		}
-
 		$separator = '&';
 
 		$urlQuery = $separator . $urlQuery;
@@ -891,7 +890,7 @@ static public function getRequestVar($varName, $varDefault = null, $varType = nu
 		{
 			if( is_null($varDefault))
 			{
-				throw new Exception("The parameter '$varName' isn't set in the Request, and a default value wasn't provided.");
+				throw new Exception("The parameter '$varName' isn't set in the Request, and a default value wasn't provided." );
 			}
 			else
 			{
@@ -1942,3 +1941,23 @@ function destroy(&$var)
 	unset($var);
 	$var = null;
 }
+
+if(!function_exists('printDebug'))
+{
+	function printDebug( $info = '' )
+	{
+		if(isset($GLOBALS['PIWIK_TRACKER_DEBUG']) && $GLOBALS['PIWIK_TRACKER_DEBUG'])
+		{
+			if(is_array($info))
+			{
+				print("<pre>");
+				print(htmlspecialchars(var_export($info,true), ENT_QUOTES));
+				print("</pre>");
+			}
+			else
+			{
+				print(htmlspecialchars($info, ENT_QUOTES) . "<br />\n");
+			}
+		}
+	}
+}

core/DataTable/Filter/AddSummaryRow.php

@@ -77,12 +77,12 @@ public function filter($table)
 				//FIXME: I'm not sure why it could return false, but it was reported in: http://forum.piwik.org/read.php?2,89324,page=1#msg-89442
 				if($summaryRow)
 				{
-					$newRow->sumRow($summaryRow);
+					$newRow->sumRow($summaryRow, $enableCopyMetadata = false);
 				}
 			}
 			else
 			{
-				$newRow->sumRow($rows[$i]);
+				$newRow->sumRow($rows[$i], $enableCopyMetadata = false);
 			}
 		}
 

core/DataTable/Filter/ColumnDelete.php

@@ -33,6 +33,13 @@ class Piwik_DataTable_Filter_ColumnDelete extends Piwik_DataTable_Filter
 	 * @var array
 	 */
 	private $columnsToKeep;
+
+	/**
+	 * Delete the column, only if the value was zero
+	 *
+	 * @var bool
+	 */
+	private $deleteIfZeroOnly;
 
 	/**
 	 * Constructor.
@@ -44,7 +51,7 @@ class Piwik_DataTable_Filter_ColumnDelete extends Piwik_DataTable_Filter
 	 *                                    comma-separated list of column names. Columns not in
 	 *                                    this list will be removed.
 	 */
-	public function __construct( $table, $columnsToRemove, $columnsToKeep = array() )
+	public function __construct( $table, $columnsToRemove, $columnsToKeep = array(), $deleteIfZeroOnly = false )
 	{
 		parent::__construct($table);
 
@@ -60,28 +67,36 @@ public function __construct( $table, $columnsToRemove, $columnsToKeep = array()
 
 		$this->columnsToRemove = $columnsToRemove;
 		$this->columnsToKeep = array_flip($columnsToKeep); // flip so we can use isset instead of in_array
-
-		// always do recursive filter
-		$this->enableRecursive(true);
+		$this->deleteIfZeroOnly = $deleteIfZeroOnly;
 	}
-	
+
 	/**
 	 * Filters the given DataTable. Removes columns that are not desired from
 	 * each DataTable row.
-	 * 
+	 *
 	 * @param Piwik_DataTable $table
 	 */
 	public function filter($table)
 	{
+		// always do recursive filter
+		$this->enableRecursive(true);
 		$recurse = false; // only recurse if there are columns to remove/keep
-		
+
 		// remove columns specified in $this->columnsToRemove
 		if (!empty($this->columnsToRemove))
 		{
 			foreach ($table->getRows() as $row)
 			{
 				foreach ($this->columnsToRemove as $column)
 				{
+					if($this->deleteIfZeroOnly)
+					{
+						$value = $row->getColumn($column);
+						if($value === false || !empty($value))
+						{
+							continue;
+						}
+					}
 					$row->deleteColumn($column);
 				}
 			}

core/Db/Schema/Myisam.php

@@ -82,6 +82,9 @@ public function getTablesCreateSql()
 						  main_url VARCHAR(255) NOT NULL,
   						  ts_created TIMESTAMP NULL,
   						  ecommerce TINYINT DEFAULT 0,
+  						  sitesearch TINYINT DEFAULT 1,
+  						  sitesearch_keyword_parameters TEXT NOT NULL,
+  						  sitesearch_category_parameters TEXT NOT NULL,
   						  timezone VARCHAR( 50 ) NOT NULL,
   						  currency CHAR( 3 ) NOT NULL,
   						  excluded_ips TEXT NOT NULL,
@@ -187,6 +190,7 @@ class_name VARCHAR(255) NULL,
 							  visit_entry_idaction_url INTEGER(11) UNSIGNED NOT NULL,
 							  visit_entry_idaction_name INTEGER(11) UNSIGNED NOT NULL,
 							  visit_total_actions SMALLINT(5) UNSIGNED NOT NULL,
+							  visit_total_searches SMALLINT(5) UNSIGNED NOT NULL,
 							  visit_total_time SMALLINT(5) UNSIGNED NOT NULL,
 							  visit_goal_converted TINYINT(1) NOT NULL,
 							  visit_goal_buyer TINYINT(1) NOT NULL, 
@@ -310,7 +314,7 @@ class_name VARCHAR(255) NULL,
 									  		  idvisitor BINARY(8) NOT NULL,
 									          server_time DATETIME NOT NULL,
 											  idvisit INTEGER(10) UNSIGNED NOT NULL,
-											  idaction_url INTEGER(10) UNSIGNED NOT NULL,
+											  idaction_url INTEGER(10) UNSIGNED DEFAULT NULL,
 											  idaction_url_ref INTEGER(10) UNSIGNED NOT NULL,
 											  idaction_name INTEGER(10) UNSIGNED,
 											  idaction_name_ref INTEGER(10) UNSIGNED NOT NULL,

core/Piwik.php

@@ -2453,6 +2453,7 @@ static public function createTableFromCSVFile($tableName, $fields, $filePath, $f
 
 				return true;
 			} catch(Exception $e) {
+//				echo $sql . ' ---- ' .  $e->getMessage();
 				if(!Zend_Registry::get('db')->isErrNo($e, '1148'))
 				{
 					Piwik::log("LOAD DATA INFILE failed... Error was:" . $e->getMessage());
@@ -2498,16 +2499,23 @@ static public function tableInsertBatch($tableName, $fields, $values)
 
 				self::createCSVFile($filePath, $fileSpec, $values);
 
+				if(!file_exists($filePath))
+				{
+					throw new Exception("File $filePath could not be created.");
+				}
 				$rc = self::createTableFromCSVFile($tableName, $fields, $filePath, $fileSpec);
 				if($rc)
 				{
 					unlink($filePath);
 					return true;
 				}
-
-				throw new Exception('unknown cause');
+				else
+				{
+					throw new Exception('unknown cause');
+				}
 
 			} catch(Exception $e) {
+//				echo "LOAD DATA INFILE failed or not supported, falling back to normal INSERTs... Error was:" . $e->getMessage();
 				Piwik::log("LOAD DATA INFILE failed or not supported, falling back to normal INSERTs... Error was:" . $e->getMessage());
 			}
 		}

core/PluginsFunctions/WidgetsList.php

@@ -70,6 +70,7 @@ protected static function _sortWidgetCategories($a, $b)
 			'General_Visitors',
 			'UserSettings_VisitorSettings',
 			'Actions_Actions',
+			'Actions_SubmenuSitesearch',
 			'Referers_Referers',
 			'Goals_Goals',
 			'Goals_Ecommerce',

core/ScheduledTask.php

@@ -23,7 +23,7 @@ class Piwik_ScheduledTask
 	const NORMAL_PRIORITY 	= 6;
 	const HIGH_PRIORITY 	= 3;
 	const HIGHEST_PRIORITY 	= 0;
-	
+
 	/**
 	 * Class name where the specified method is located
 	 * @var string 

core/Site.php

@@ -184,6 +184,26 @@ function isEcommerceEnabled()
 		return $this->get('ecommerce') == 1;
 	}
 
+	function getSearchKeywordParameters()
+	{
+		return $this->get('sitesearch_keyword_parameters');
+	}
+
+	function getSearchCategoryParameters()
+	{
+		return $this->get('sitesearch_category_parameters');
+	}
+
+	/**
+	 * Returns whether Site Search Tracking is enabled for the site
+	 *
+	 * @return bool
+	 */
+	function isSiteSearchEnabled()
+	{
+		return $this->get('sitesearch') == 1;
+	}
+
 	/**
 	 * Checks the given string for valid site ids and returns them as an array
 	 *
@@ -283,10 +303,10 @@ static public function getMainUrlFor($idsite)
 	{
 		return self::getFor($idsite, 'main_url');
 	}
-	
+
 	/**
 	 * Returns whether the site with the specified ID is ecommerce enabled
-	 * 
+	 *
 	 * @param int  $idsite  The site ID.
 	 * @return string
 	 */
@@ -295,6 +315,17 @@ static public function isEcommerceEnabledFor($idsite)
 		return self::getFor($idsite, 'ecommerce') == 1;
 	}
 
+	/**
+	 * Returns whether the site with the specified ID is Site Search enabled
+	 *
+	 * @param int  $idsite  The site ID.
+	 * @return string
+	 */
+	static public function isSiteSearchEnabledFor($idsite)
+	{
+		return self::getFor($idsite, 'sitesearch') == 1;
+	}
+
 	/**
 	 * Returns the currency of the site with the specified ID.
 	 * 

core/Tracker.php

@@ -707,26 +707,6 @@ public static function setTestEnvironment( $args = null, $requestMethod = null )
 	}
 }
 
-if(!function_exists('printDebug'))
-{
-	function printDebug( $info = '' )
-	{
-		if(isset($GLOBALS['PIWIK_TRACKER_DEBUG']) && $GLOBALS['PIWIK_TRACKER_DEBUG'])
-		{
-			if(is_array($info))
-			{
-				print("<pre>");
-				print(htmlspecialchars(var_export($info,true), ENT_QUOTES));
-				print("</pre>");
-			}
-			else
-			{
-				print(htmlspecialchars($info, ENT_QUOTES) . "<br />\n");
-			}
-		}
-	}
-}
-
 if(!function_exists('Piwik_Translate'))
 {
 	function Piwik_Translate($string, $args = array())