Skip to content

Commit

Permalink
Fixes #3832, add --force-lowercase-path option to log importer script…
Browse files Browse the repository at this point in the history 
… that makes all paths lowercase before importing hits.
  • Loading branch information
diosmosis committed Apr 18, 2013
1 parent 98658d0 commit bf2fdd4
Show file tree
Hide file tree
Showing 44 changed files with 304 additions and 295 deletions.
8 changes: 8 additions & 0 deletions misc/log-analytics/import_logs.py
View file
Expand Up @@ -399,6 +399,11 @@ def _create_parser(self):
help="Invalidate reports for the specified dates (format: YYYY-MM-DD,YYYY-MM-DD,...). "
"By default, all dates found in the logs will be invalidated.",
)
option_parser.add_option(
'--force-lowercase-path', dest='force_lowercase_path', default=False, action='store_true',
help="Make URL path lowercase so paths with the same letters but different cases are "
"treated the same."
)
return option_parser


Expand Down Expand Up @@ -1225,6 +1230,9 @@ def __init__(self, **kwargs):
for key, value in kwargs.iteritems():
setattr(self, key, value)
super(Hit, self).__init__()

if config.options.force_lowercase_path:
self.full_path = self.full_path.lower()


class Parser(object):
Expand Down
3 changes: 2 additions & 1 deletion tests/PHPUnit/Fixtures/ManySitesImportedLogs.php
View file
Expand Up @@ -99,7 +99,8 @@ private function logVisitsWithAllEnabled()
'--enable-bots' => false,
'--enable-http-errors' => false,
'--enable-http-redirects' => false,
'--enable-reverse-dns' => false);
'--enable-reverse-dns' => false,
'--force-lowercase-path' => false);

self::executeLogImporter($logFile, $opts);
}
Expand Down
54 changes: 27 additions & 27 deletions tests/PHPUnit/Integration/expected/test_ImportLogs__Actions.getEntryPageUrls_month.xml
View file
Expand Up @@ -2,59 +2,59 @@
<result>
<row>
<label>blog</label>
<nb_visits>4</nb_visits>
<nb_hits>4</nb_hits>
<nb_visits>5</nb_visits>
<nb_hits>5</nb_hits>
<sum_time_spent>0</sum_time_spent>
<entry_nb_visits>4</entry_nb_visits>
<entry_nb_actions>4</entry_nb_actions>
<entry_nb_visits>5</entry_nb_visits>
<entry_nb_actions>5</entry_nb_actions>
<entry_sum_visit_length>0</entry_sum_visit_length>
<entry_bounce_count>4</entry_bounce_count>
<exit_nb_visits>4</exit_nb_visits>
<entry_bounce_count>5</entry_bounce_count>
<exit_nb_visits>5</exit_nb_visits>
<avg_time_on_page>0</avg_time_on_page>
<bounce_rate>100%</bounce_rate>
<exit_rate>100%</exit_rate>
<subtable>
<row>
<label>category</label>
<nb_visits>3</nb_visits>
<nb_hits>3</nb_hits>
<nb_visits>4</nb_visits>
<nb_hits>4</nb_hits>
<sum_time_spent>0</sum_time_spent>
<entry_nb_visits>3</entry_nb_visits>
<entry_nb_actions>3</entry_nb_actions>
<entry_nb_visits>4</entry_nb_visits>
<entry_nb_actions>4</entry_nb_actions>
<entry_sum_visit_length>0</entry_sum_visit_length>
<entry_bounce_count>3</entry_bounce_count>
<exit_nb_visits>3</exit_nb_visits>
<entry_bounce_count>4</entry_bounce_count>
<exit_nb_visits>4</exit_nb_visits>
<avg_time_on_page>0</avg_time_on_page>
<bounce_rate>100%</bounce_rate>
<exit_rate>100%</exit_rate>
<subtable>
<row>
<label>meta</label>
<nb_visits>2</nb_visits>
<nb_hits>2</nb_hits>
<nb_visits>3</nb_visits>
<nb_hits>3</nb_hits>
<sum_time_spent>0</sum_time_spent>
<entry_nb_visits>2</entry_nb_visits>
<entry_nb_actions>2</entry_nb_actions>
<entry_nb_visits>3</entry_nb_visits>
<entry_nb_actions>3</entry_nb_actions>
<entry_sum_visit_length>0</entry_sum_visit_length>
<entry_bounce_count>2</entry_bounce_count>
<exit_nb_visits>2</exit_nb_visits>
<entry_bounce_count>3</entry_bounce_count>
<exit_nb_visits>3</exit_nb_visits>
<avg_time_on_page>0</avg_time_on_page>
<bounce_rate>100%</bounce_rate>
<exit_rate>100%</exit_rate>
<subtable>
<row>
<label>/index</label>
<nb_visits>2</nb_visits>
<nb_hits>2</nb_hits>
<nb_visits>3</nb_visits>
<nb_hits>3</nb_hits>
<sum_time_spent>0</sum_time_spent>
<entry_nb_visits>2</entry_nb_visits>
<entry_nb_actions>2</entry_nb_actions>
<entry_nb_visits>3</entry_nb_visits>
<entry_nb_actions>3</entry_nb_actions>
<entry_sum_visit_length>0</entry_sum_visit_length>
<entry_bounce_count>2</entry_bounce_count>
<exit_nb_visits>2</exit_nb_visits>
<sum_daily_nb_uniq_visitors>2</sum_daily_nb_uniq_visitors>
<sum_daily_entry_nb_uniq_visitors>2</sum_daily_entry_nb_uniq_visitors>
<sum_daily_exit_nb_uniq_visitors>2</sum_daily_exit_nb_uniq_visitors>
<entry_bounce_count>3</entry_bounce_count>
<exit_nb_visits>3</exit_nb_visits>
<sum_daily_nb_uniq_visitors>3</sum_daily_nb_uniq_visitors>
<sum_daily_entry_nb_uniq_visitors>3</sum_daily_entry_nb_uniq_visitors>
<sum_daily_exit_nb_uniq_visitors>3</sum_daily_exit_nb_uniq_visitors>
<avg_time_on_page>0</avg_time_on_page>
<bounce_rate>100%</bounce_rate>
<exit_rate>100%</exit_rate>
Expand Down
54 changes: 27 additions & 27 deletions tests/PHPUnit/Integration/expected/test_ImportLogs__Actions.getExitPageUrls_month.xml
View file
Expand Up @@ -2,59 +2,59 @@
<result>
<row>
<label>blog</label>
<nb_visits>4</nb_visits>
<nb_hits>4</nb_hits>
<nb_visits>5</nb_visits>
<nb_hits>5</nb_hits>
<sum_time_spent>0</sum_time_spent>
<entry_nb_visits>4</entry_nb_visits>
<entry_nb_actions>4</entry_nb_actions>
<entry_nb_visits>5</entry_nb_visits>
<entry_nb_actions>5</entry_nb_actions>
<entry_sum_visit_length>0</entry_sum_visit_length>
<entry_bounce_count>4</entry_bounce_count>
<exit_nb_visits>4</exit_nb_visits>
<entry_bounce_count>5</entry_bounce_count>
<exit_nb_visits>5</exit_nb_visits>
<avg_time_on_page>0</avg_time_on_page>
<bounce_rate>100%</bounce_rate>
<exit_rate>100%</exit_rate>
<subtable>
<row>
<label>category</label>
<nb_visits>3</nb_visits>
<nb_hits>3</nb_hits>
<nb_visits>4</nb_visits>
<nb_hits>4</nb_hits>
<sum_time_spent>0</sum_time_spent>
<entry_nb_visits>3</entry_nb_visits>
<entry_nb_actions>3</entry_nb_actions>
<entry_nb_visits>4</entry_nb_visits>
<entry_nb_actions>4</entry_nb_actions>
<entry_sum_visit_length>0</entry_sum_visit_length>
<entry_bounce_count>3</entry_bounce_count>
<exit_nb_visits>3</exit_nb_visits>
<entry_bounce_count>4</entry_bounce_count>
<exit_nb_visits>4</exit_nb_visits>
<avg_time_on_page>0</avg_time_on_page>
<bounce_rate>100%</bounce_rate>
<exit_rate>100%</exit_rate>
<subtable>
<row>
<label>meta</label>
<nb_visits>2</nb_visits>
<nb_hits>2</nb_hits>
<nb_visits>3</nb_visits>
<nb_hits>3</nb_hits>
<sum_time_spent>0</sum_time_spent>
<entry_nb_visits>2</entry_nb_visits>
<entry_nb_actions>2</entry_nb_actions>
<entry_nb_visits>3</entry_nb_visits>
<entry_nb_actions>3</entry_nb_actions>
<entry_sum_visit_length>0</entry_sum_visit_length>
<entry_bounce_count>2</entry_bounce_count>
<exit_nb_visits>2</exit_nb_visits>
<entry_bounce_count>3</entry_bounce_count>
<exit_nb_visits>3</exit_nb_visits>
<avg_time_on_page>0</avg_time_on_page>
<bounce_rate>100%</bounce_rate>
<exit_rate>100%</exit_rate>
<subtable>
<row>
<label>/index</label>
<nb_visits>2</nb_visits>
<nb_hits>2</nb_hits>
<nb_visits>3</nb_visits>
<nb_hits>3</nb_hits>
<sum_time_spent>0</sum_time_spent>
<entry_nb_visits>2</entry_nb_visits>
<entry_nb_actions>2</entry_nb_actions>
<entry_nb_visits>3</entry_nb_visits>
<entry_nb_actions>3</entry_nb_actions>
<entry_sum_visit_length>0</entry_sum_visit_length>
<entry_bounce_count>2</entry_bounce_count>
<exit_nb_visits>2</exit_nb_visits>
<sum_daily_nb_uniq_visitors>2</sum_daily_nb_uniq_visitors>
<sum_daily_entry_nb_uniq_visitors>2</sum_daily_entry_nb_uniq_visitors>
<sum_daily_exit_nb_uniq_visitors>2</sum_daily_exit_nb_uniq_visitors>
<entry_bounce_count>3</entry_bounce_count>
<exit_nb_visits>3</exit_nb_visits>
<sum_daily_nb_uniq_visitors>3</sum_daily_nb_uniq_visitors>
<sum_daily_entry_nb_uniq_visitors>3</sum_daily_entry_nb_uniq_visitors>
<sum_daily_exit_nb_uniq_visitors>3</sum_daily_exit_nb_uniq_visitors>
<avg_time_on_page>0</avg_time_on_page>
<bounce_rate>100%</bounce_rate>
<exit_rate>100%</exit_rate>
Expand Down
6 changes: 3 additions & 3 deletions tests/PHPUnit/Integration/expected/test_ImportLogs__Actions.getPageTitles_month.xml
View file
Expand Up @@ -2,10 +2,10 @@
<result>
<row>
<label>Page Name not defined</label>
<nb_visits>18</nb_visits>
<nb_hits>18</nb_hits>
<nb_visits>19</nb_visits>
<nb_hits>19</nb_hits>
<sum_time_spent>0</sum_time_spent>
<sum_daily_nb_uniq_visitors>18</sum_daily_nb_uniq_visitors>
<sum_daily_nb_uniq_visitors>19</sum_daily_nb_uniq_visitors>
<avg_time_on_page>0</avg_time_on_page>
<bounce_rate>0%</bounce_rate>
<exit_rate>0%</exit_rate>
Expand Down
54 changes: 27 additions & 27 deletions tests/PHPUnit/Integration/expected/test_ImportLogs__Actions.getPageUrls_month.xml
View file
Expand Up @@ -2,59 +2,59 @@
<result>
<row>
<label>blog</label>
<nb_visits>4</nb_visits>
<nb_hits>4</nb_hits>
<nb_visits>5</nb_visits>
<nb_hits>5</nb_hits>
<sum_time_spent>0</sum_time_spent>
<entry_nb_visits>4</entry_nb_visits>
<entry_nb_actions>4</entry_nb_actions>
<entry_nb_visits>5</entry_nb_visits>
<entry_nb_actions>5</entry_nb_actions>
<entry_sum_visit_length>0</entry_sum_visit_length>
<entry_bounce_count>4</entry_bounce_count>
<exit_nb_visits>4</exit_nb_visits>
<entry_bounce_count>5</entry_bounce_count>
<exit_nb_visits>5</exit_nb_visits>
<avg_time_on_page>0</avg_time_on_page>
<bounce_rate>100%</bounce_rate>
<exit_rate>100%</exit_rate>
<subtable>
<row>
<label>category</label>
<nb_visits>3</nb_visits>
<nb_hits>3</nb_hits>
<nb_visits>4</nb_visits>
<nb_hits>4</nb_hits>
<sum_time_spent>0</sum_time_spent>
<entry_nb_visits>3</entry_nb_visits>
<entry_nb_actions>3</entry_nb_actions>
<entry_nb_visits>4</entry_nb_visits>
<entry_nb_actions>4</entry_nb_actions>
<entry_sum_visit_length>0</entry_sum_visit_length>
<entry_bounce_count>3</entry_bounce_count>
<exit_nb_visits>3</exit_nb_visits>
<entry_bounce_count>4</entry_bounce_count>
<exit_nb_visits>4</exit_nb_visits>
<avg_time_on_page>0</avg_time_on_page>
<bounce_rate>100%</bounce_rate>
<exit_rate>100%</exit_rate>
<subtable>
<row>
<label>meta</label>
<nb_visits>2</nb_visits>
<nb_hits>2</nb_hits>
<nb_visits>3</nb_visits>
<nb_hits>3</nb_hits>
<sum_time_spent>0</sum_time_spent>
<entry_nb_visits>2</entry_nb_visits>
<entry_nb_actions>2</entry_nb_actions>
<entry_nb_visits>3</entry_nb_visits>
<entry_nb_actions>3</entry_nb_actions>
<entry_sum_visit_length>0</entry_sum_visit_length>
<entry_bounce_count>2</entry_bounce_count>
<exit_nb_visits>2</exit_nb_visits>
<entry_bounce_count>3</entry_bounce_count>
<exit_nb_visits>3</exit_nb_visits>
<avg_time_on_page>0</avg_time_on_page>
<bounce_rate>100%</bounce_rate>
<exit_rate>100%</exit_rate>
<subtable>
<row>
<label>/index</label>
<nb_visits>2</nb_visits>
<nb_hits>2</nb_hits>
<nb_visits>3</nb_visits>
<nb_hits>3</nb_hits>
<sum_time_spent>0</sum_time_spent>
<entry_nb_visits>2</entry_nb_visits>
<entry_nb_actions>2</entry_nb_actions>
<entry_nb_visits>3</entry_nb_visits>
<entry_nb_actions>3</entry_nb_actions>
<entry_sum_visit_length>0</entry_sum_visit_length>
<entry_bounce_count>2</entry_bounce_count>
<exit_nb_visits>2</exit_nb_visits>
<sum_daily_nb_uniq_visitors>2</sum_daily_nb_uniq_visitors>
<sum_daily_entry_nb_uniq_visitors>2</sum_daily_entry_nb_uniq_visitors>
<sum_daily_exit_nb_uniq_visitors>2</sum_daily_exit_nb_uniq_visitors>
<entry_bounce_count>3</entry_bounce_count>
<exit_nb_visits>3</exit_nb_visits>
<sum_daily_nb_uniq_visitors>3</sum_daily_nb_uniq_visitors>
<sum_daily_entry_nb_uniq_visitors>3</sum_daily_entry_nb_uniq_visitors>
<sum_daily_exit_nb_uniq_visitors>3</sum_daily_exit_nb_uniq_visitors>
<avg_time_on_page>0</avg_time_on_page>
<bounce_rate>100%</bounce_rate>
<exit_rate>100%</exit_rate>
Expand Down
4 changes: 2 additions & 2 deletions tests/PHPUnit/Integration/expected/test_ImportLogs__Actions.get_month.xml
View file
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="utf-8" ?>
<result>
<nb_pageviews>24</nb_pageviews>
<nb_uniq_pageviews>24</nb_uniq_pageviews>
<nb_pageviews>25</nb_pageviews>
<nb_uniq_pageviews>25</nb_uniq_pageviews>
<nb_downloads>4</nb_downloads>
<nb_uniq_downloads>4</nb_uniq_downloads>
<nb_outlinks>0</nb_outlinks>
Expand Down

0 comments on commit bf2fdd4

Please sign in to comment.