Skip to content
This repository
Newer
Older
100644 358 lines (324 sloc) 11.724 kb
995c6623 » matt
2008-08-04 oops i totally screwed up my last commit, deleting /modules instead o…
1 <?php
2 /**
3 * Piwik - Open source web analytics
4 *
5 * @link http://piwik.org
49ee5fe9 » vipsoft
2010-08-20 refs #1630
6 * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
e99e2234 » vipsoft
2009-07-01 Add svn:keywords Id to .php files in core, plugins, and js
7 * @version $Id$
995c6623 » matt
2008-08-04 oops i totally screwed up my last commit, deleting /modules instead o…
8 *
17648f51 » vipsoft
2009-08-22 phpdoc cleanup:
9 * @category Piwik
10 * @package Piwik
995c6623 » matt
2008-08-04 oops i totally screwed up my last commit, deleting /modules instead o…
11 */
12
13 /**
14 * Handles the archiving process for a period
15 *
16 * This class provides generic methods to archive data for a period (week / month / year).
17 *
18 * These methods are called by the plugins that do the logic of archiving their own data. \
19 * They hook on the event 'ArchiveProcessing_Period.compute'
20 *
17648f51 » vipsoft
2009-08-22 phpdoc cleanup:
21 * @package Piwik
22 * @subpackage Piwik_ArchiveProcessing
995c6623 » matt
2008-08-04 oops i totally screwed up my last commit, deleting /modules instead o…
23 */
24 class Piwik_ArchiveProcessing_Period extends Piwik_ArchiveProcessing
25 {
3412d1bd » matt
2009-04-03 - changing ways of applying filters to a datatable, now $table->filte…
26 /*
27 * Array of (column name before => column name renamed) of the columns for which sum operation is invalid.
28 * The summed value is not accurate and these columns will be renamed accordingly.
29 */
30 static public $invalidSummedColumnNameToRenamedName = array(
31 Piwik_Archive::INDEX_NB_UNIQ_VISITORS => Piwik_Archive::INDEX_SUM_DAILY_NB_UNIQ_VISITORS
32 );
33
bc89b5ec » matt
2009-03-27 - memory and speed optimizations of archiving, refs #374 (hopefully f…
34 public function __construct()
35 {
36 parent::__construct();
37 $this->debugAlwaysArchive = Zend_Registry::get('config')->Debug->always_archive_data_period;
38 }
39
995c6623 » matt
2008-08-04 oops i totally screwed up my last commit, deleting /modules instead o…
40 /**
41 * Sums all values for the given field names $aNames over the period
42 * See @archiveNumericValuesGeneral for more information
43 *
44 * @param string|array
45 * @return Piwik_ArchiveProcessing_Record_Numeric
46 *
47 */
48 public function archiveNumericValuesSum( $aNames )
49 {
50 return $this->archiveNumericValuesGeneral($aNames, 'sum');
51 }
52
53 /**
54 * Get the maximum value for all values for the given field names $aNames over the period
55 * See @archiveNumericValuesGeneral for more information
56 *
57 * @param string|array
58 * @return Piwik_ArchiveProcessing_Record_Numeric
59 *
60 */
61 public function archiveNumericValuesMax( $aNames )
62 {
63 return $this->archiveNumericValuesGeneral($aNames, 'max');
64 }
65
66 /**
67 * Given a list of fields names, the method will fetch all their values over the period, and archive them using the given operation.
68 *
69 * For example if $operationToApply = 'sum' and $aNames = array('nb_visits', 'sum_time_visit')
70 * it will sum all values of nb_visits for the period (for example give the number of visits for the month by summing the visits of every day)
71 *
72 * @param array|string $aNames Array of strings or string containg the field names to select
73 * @param string $operationToApply Available operations = sum, max, min
74 * @return Piwik_ArchiveProcessing_Record_Numeric Returns the record if $aNames is a string,
75 * an array of Piwik_ArchiveProcessing_Record_Numeric indexed by their field names if aNames is an array of strings
76 */
77 private function archiveNumericValuesGeneral($aNames, $operationToApply)
78 {
79 if(!is_array($aNames))
80 {
81 $aNames = array($aNames);
82 }
83
84 // fetch the numeric values and apply the operation on them
85 $results = array();
86 foreach($this->archives as $archive)
87 {
88 foreach($aNames as $name)
89 {
90 if(!isset($results[$name]))
91 {
92 $results[$name] = 0;
93 }
94 $valueToSum = $archive->getNumeric($name);
95
96 if($valueToSum !== false)
97 {
98 switch ($operationToApply) {
99 case 'sum':
100 $results[$name] += $valueToSum;
101 break;
102 case 'max':
103 $results[$name] = max($results[$name], $valueToSum);
104 break;
105 case 'min':
106 $results[$name] = min($results[$name], $valueToSum);
107 break;
108 default:
109 throw new Exception("Operation not applicable.");
110 break;
111 }
112 }
113 }
114 }
115
116 // build the Record Numeric objects
117 $records = array();
118 foreach($results as $name => $value)
119 {
b9f35b21 » mauser
2009-07-09 fixes #842 computing the real number of unique visitors per month and…
120 if($name == 'nb_uniq_visitors' && ($this->periodId == Piwik::$idPeriods['week'] || $this->periodId == Piwik::$idPeriods['month']))
121 {
122 $value = (float) $this->computeNbUniqVisitors();
123 }
995c6623 » matt
2008-08-04 oops i totally screwed up my last commit, deleting /modules instead o…
124 $records[$name] = new Piwik_ArchiveProcessing_Record_Numeric(
125 $name,
126 $value
127 );
c8af92a2 » matt
2009-03-26 - fixing broken unit tests and previously broken period archiving.
128 $this->insertRecord($records[$name]);
995c6623 » matt
2008-08-04 oops i totally screwed up my last commit, deleting /modules instead o…
129 }
130
131 // if asked for only one field to sum
132 if(count($records) == 1)
133 {
134 return $records[$name];
135 }
136
137 // returns the array of records once summed
138 return $records;
139 }
140
141
142 /**
3966931c » matt
2008-12-17 - adding Goal Tracking related goodness in core, and plugins
143 * This method will compute the sum of DataTables over the period for the given fields $aRecordName.
995c6623 » matt
2008-08-04 oops i totally screwed up my last commit, deleting /modules instead o…
144 * The resulting DataTable will be then added to queue of data to be recorded in the database.
145 * It will usually be called in a plugin that listens to the hook 'ArchiveProcessing_Period.compute'
146 *
147 * For example if $aRecordName = 'UserCountry_country' the method will select all UserCountry_country DataTable for the period
63022a14 » matt
2009-03-10 - refs #374 Performance: Fix memory leak during Archiving process
148 * (eg. the 31 dataTable of the last month), sum them, and create the Piwik_ArchiveProcessing_RecordArray so that
995c6623 » matt
2008-08-04 oops i totally screwed up my last commit, deleting /modules instead o…
149 * the resulting dataTable is AUTOMATICALLY recorded in the database.
150 *
151 *
152 * This method works on recursive dataTable. For example for the 'Actions' it will select all subtables of all dataTable of all the sub periods
153 * and get the sum.
154 *
155 * It returns an array that gives information about the "final" DataTable. The array gives for every field name, the number of rows in the
156 * final DataTable (ie. the number of distinct LABEL over the period) (eg. the number of distinct keywords over the last month)
157 *
158 * @param string|array Field name(s) of DataTable to select so we can get the sum
553420e2 » matt
2010-03-15 Fixes #1011 Provider plugin now limits row count to 500
159 * @param array (current_column_name => new_column_name) for columns that must change names when summed (eg. unique visitors go from nb_uniq_visitors to sum_daily_nb_uniq_visitors)
160 * @param int Max row count of parent datatable to archive
161 * @param int Max row count of children datatable(s) to archive
162 * @param string Column name to sort by, before truncating rows (ie. if there are more rows than the specified max row count)
163 *
995c6623 » matt
2008-08-04 oops i totally screwed up my last commit, deleting /modules instead o…
164 * @return array array (
165 * nameTable1 => number of rows,
166 * nameTable2 => number of rows,
167 * )
168 */
c8af92a2 » matt
2009-03-26 - fixing broken unit tests and previously broken period archiving.
169 public function archiveDataTable( $aRecordName,
3412d1bd » matt
2009-04-03 - changing ways of applying filters to a datatable, now $table->filte…
170 $invalidSummedColumnNameToRenamedName = null,
c8af92a2 » matt
2009-03-26 - fixing broken unit tests and previously broken period archiving.
171 $maximumRowsInDataTableLevelZero = null,
172 $maximumRowsInSubDataTable = null,
173 $columnToSortByBeforeTruncation = null )
995c6623 » matt
2008-08-04 oops i totally screwed up my last commit, deleting /modules instead o…
174 {
175 if(!is_array($aRecordName))
176 {
177 $aRecordName = array($aRecordName);
178 }
179
180 $nameToCount = array();
181 foreach($aRecordName as $recordName)
182 {
3412d1bd » matt
2009-04-03 - changing ways of applying filters to a datatable, now $table->filte…
183 $table = $this->getRecordDataTableSum($recordName, $invalidSummedColumnNameToRenamedName);
995c6623 » matt
2008-08-04 oops i totally screwed up my last commit, deleting /modules instead o…
184
185 $nameToCount[$recordName]['level0'] = $table->getRowsCount();
186 $nameToCount[$recordName]['recursive'] = $table->getRowsCountRecursive();
187
c8af92a2 » matt
2009-03-26 - fixing broken unit tests and previously broken period archiving.
188 $blob = $table->getSerialized( $maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable, $columnToSortByBeforeTruncation );
bc89b5ec » matt
2009-03-27 - memory and speed optimizations of archiving, refs #374 (hopefully f…
189 destroy($table);
c8af92a2 » matt
2009-03-26 - fixing broken unit tests and previously broken period archiving.
190 $this->insertBlobRecord($recordName, $blob);
995c6623 » matt
2008-08-04 oops i totally screwed up my last commit, deleting /modules instead o…
191 }
bc89b5ec » matt
2009-03-27 - memory and speed optimizations of archiving, refs #374 (hopefully f…
192 Piwik_DataTable_Manager::getInstance()->deleteAll();
193
995c6623 » matt
2008-08-04 oops i totally screwed up my last commit, deleting /modules instead o…
194 return $nameToCount;
195 }
bc89b5ec » matt
2009-03-27 - memory and speed optimizations of archiving, refs #374 (hopefully f…
196
995c6623 » matt
2008-08-04 oops i totally screwed up my last commit, deleting /modules instead o…
197 /**
198 * This method selects all DataTables that have the name $name over the period.
199 * It calls the appropriate methods that sum all these tables together.
200 * The resulting DataTable is returned.
201 *
202 * @param string $name
3412d1bd » matt
2009-04-03 - changing ways of applying filters to a datatable, now $table->filte…
203 * @param array columns in the array (old name, new name) to be renamed as the sum operation is not valid on them (eg. nb_uniq_visitors->sum_daily_nb_uniq_visitors)
995c6623 » matt
2008-08-04 oops i totally screwed up my last commit, deleting /modules instead o…
204 * @return Piwik_DataTable
205 */
3412d1bd » matt
2009-04-03 - changing ways of applying filters to a datatable, now $table->filte…
206 protected function getRecordDataTableSum( $name, $invalidSummedColumnNameToRenamedName )
995c6623 » matt
2008-08-04 oops i totally screwed up my last commit, deleting /modules instead o…
207 {
9fa70ba6 » vipsoft
2009-07-23 Add () when instantiating classes for consistency.
208 $table = new Piwik_DataTable();
995c6623 » matt
2008-08-04 oops i totally screwed up my last commit, deleting /modules instead o…
209 foreach($this->archives as $archive)
210 {
211 $archive->preFetchBlob($name);
212 $datatableToSum = $archive->getDataTable($name);
213 $archive->loadSubDataTables($name, $datatableToSum);
214 $table->addDataTable($datatableToSum);
215 $archive->freeBlob($name);
216 }
3412d1bd » matt
2009-04-03 - changing ways of applying filters to a datatable, now $table->filte…
217
218 if(is_null($invalidSummedColumnNameToRenamedName))
219 {
220 $invalidSummedColumnNameToRenamedName = self::$invalidSummedColumnNameToRenamedName;
221 }
222 foreach($invalidSummedColumnNameToRenamedName as $oldName => $newName)
223 {
224 $table->renameColumn($oldName, $newName);
225 }
995c6623 » matt
2008-08-04 oops i totally screwed up my last commit, deleting /modules instead o…
226 return $table;
227 }
228
229 protected function initCompute()
230 {
231 parent::initCompute();
232 $this->archives = $this->loadSubperiodsArchive();
233 }
234
235 /**
236 * Returns the ID of the archived subperiods.
237 *
238 * @return array Array of the idArchive of the subperiods
239 */
240 protected function loadSubperiodsArchive()
241 {
242 $periods = array();
243
244 // we first compute every subperiod of the archive
245 foreach($this->period->getSubperiods() as $period)
246 {
9fa70ba6 » vipsoft
2009-07-23 Add () when instantiating classes for consistency.
247 $archivePeriod = new Piwik_Archive_Single();
995c6623 » matt
2008-08-04 oops i totally screwed up my last commit, deleting /modules instead o…
248 $archivePeriod->setSite( $this->site );
249 $archivePeriod->setPeriod( $period );
250 $archivePeriod->prepareArchive();
251
252 $periods[] = $archivePeriod;
253 }
254 return $periods;
255 }
256
257 /**
258 * Main method to process logs for a period.
259 * The only logic done here is computing the number of visits, actions, etc.
260 *
261 * All the other reports are computed inside plugins listening to the event 'ArchiveProcessing_Period.compute'.
262 * See some of the plugins for an example.
263 */
264 protected function compute()
265 {
266 $this->archiveNumericValuesMax( 'max_actions' );
267 $toSum = array(
d7670580 » matt
2009-03-06 - removing TODO from code and taking necessary actions
268 'nb_uniq_visitors',
995c6623 » matt
2008-08-04 oops i totally screwed up my last commit, deleting /modules instead o…
269 'nb_visits',
270 'nb_actions',
271 'sum_visit_length',
272 'bounce_count',
3966931c » matt
2008-12-17 - adding Goal Tracking related goodness in core, and plugins
273 'nb_visits_converted',
995c6623 » matt
2008-08-04 oops i totally screwed up my last commit, deleting /modules instead o…
274 );
275 $record = $this->archiveNumericValuesSum($toSum);
276
3966931c » matt
2008-12-17 - adding Goal Tracking related goodness in core, and plugins
277 $nbVisits = $record['nb_visits']->value;
278 $nbVisitsConverted = $record['nb_visits_converted']->value;
bfba0e7e » matt
2009-04-27 - API CHANGE: the API for the function Piwik_AddWidget has changed. T…
279 $this->isThereSomeVisits = ( $nbVisits > 0);
995c6623 » matt
2008-08-04 oops i totally screwed up my last commit, deleting /modules instead o…
280 if($this->isThereSomeVisits === false)
281 {
282 return;
283 }
3966931c » matt
2008-12-17 - adding Goal Tracking related goodness in core, and plugins
284 $this->setNumberOfVisits($nbVisits);
285 $this->setNumberOfVisitsConverted($nbVisitsConverted);
995c6623 » matt
2008-08-04 oops i totally screwed up my last commit, deleting /modules instead o…
286 Piwik_PostEvent('ArchiveProcessing_Period.compute', $this);
287 }
b9f35b21 » mauser
2009-07-09 fixes #842 computing the real number of unique visitors per month and…
288
289 protected function computeNbUniqVisitors()
290 {
f8932466 » matt
2010-03-29 Refs #56
291 $query = "
292 SELECT count(distinct visitor_idcookie) as nb_uniq_visitors
293 FROM ".$this->logTable."
294 WHERE visit_last_action_time >= ?
295 AND visit_last_action_time <= ?
296 AND idsite = ?";
b9f35b21 » mauser
2009-07-09 fixes #842 computing the real number of unique visitors per month and…
297
f8932466 » matt
2010-03-29 Refs #56
298 return Zend_Registry::get('db')->fetchOne($query, array( $this->getStartDatetimeUTC(), $this->getEndDatetimeUTC(), $this->idsite ));
b9f35b21 » mauser
2009-07-09 fixes #842 computing the real number of unique visitors per month and…
299 }
995c6623 » matt
2008-08-04 oops i totally screwed up my last commit, deleting /modules instead o…
300
301 /**
302 * Called at the end of the archiving process.
303 * Does some cleaning job in the database.
304 */
305 protected function postCompute()
306 {
307 parent::postCompute();
308
5fd9408b » matt
2010-04-12 Refs #56
309 foreach($this->archives as $archive)
310 {
311 destroy($archive);
312 }
313 $this->archives = array();
314
5a61b80a » matt
2009-04-07 - fixes #577 Aautomatic purge should be automatically called once a d…
315 $blobTable = $this->tableArchiveBlob->getTableName();
316 $numericTable = $this->tableArchiveNumeric->getTableName();
317
318 $key = 'lastPurge_' . $blobTable;
319 $timestamp = Piwik_GetOption($key);
320 if(!$timestamp
5fd9408b » matt
2010-04-12 Refs #56
321 || $timestamp < time() - 86400)
3966931c » matt
2008-12-17 - adding Goal Tracking related goodness in core, and plugins
322 {
5fd9408b » matt
2010-04-12 Refs #56
323 Piwik_SetOption($key, time());
f8932466 » matt
2010-03-29 Refs #56
324
5a61b80a » matt
2009-04-07 - fixes #577 Aautomatic purge should be automatically called once a d…
325 // we delete out of date daily archives from table, maximum once per day
5fd9408b » matt
2010-04-12 Refs #56
326 // we only delete archives processed that are older than 1 day, to not delete archives we just processed
327 $yesterday = Piwik_Date::factory('yesterday')->getDateTime();
328 $result = Piwik_FetchAll("
329 SELECT idarchive
330 FROM $numericTable
331 WHERE name='done'
332 AND value = ". Piwik_ArchiveProcessing::DONE_OK_TEMPORARY ."
333 AND ts_archived < ?", array($yesterday));
5a61b80a » matt
2009-04-07 - fixes #577 Aautomatic purge should be automatically called once a d…
334
5fd9408b » matt
2010-04-12 Refs #56
335 $idArchivesToDelete = array();
336 if(!empty($result))
337 {
338 foreach($result as $row) {
339 $idArchivesToDelete[] = $row['idarchive'];
340 }
341 $query = "/* SHARDING_ID_SITE = ".$this->idsite." */
342 DELETE
343 FROM %s
344 WHERE idarchive IN (".implode(',',$idArchivesToDelete).")
345 ";
346
347 Piwik_Query(sprintf($query, $blobTable));
348 Piwik_Query(sprintf($query, $numericTable));
349 }
350 Piwik::log("Purging temporary archives: done [ purged archives older than $yesterday from $blobTable and $numericTable ] [Deleted IDs: ". implode(',',$idArchivesToDelete)."]");
3966931c » matt
2008-12-17 - adding Goal Tracking related goodness in core, and plugins
351 }
5fd9408b » matt
2010-04-12 Refs #56
352 else
e02abed8 » mauser
2009-05-27 fixing core memory leaks, see also [1145]
353 {
5fd9408b » matt
2010-04-12 Refs #56
354 Piwik::log("Purging temporary archives: skipped.");
e02abed8 » mauser
2009-05-27 fixing core memory leaks, see also [1145]
355 }
5fd9408b » matt
2010-04-12 Refs #56
356
cd0cbab3 » vipsoft
2009-07-08 fixes #803 - remove unnecessary require_once from core, plugins, and …
357 }
995c6623 » matt
2008-08-04 oops i totally screwed up my last commit, deleting /modules instead o…
358 }
Something went wrong with that request. Please try again.