Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

version 1.4

  • Loading branch information...
commit 5617a533a9a1e00e8bdaf5c64302e16d2cc7fc52 1 parent 047f07c
m.sobczak authored
View
13 README
@@ -1,4 +1,4 @@
-Google2Piwik Exporter, version 1.3, August 2012
+Google2Piwik Exporter, version 1.4, October 2012
Description
===========
@@ -14,15 +14,15 @@ Requirements
However you can grant privileges to your Gmail account, and use it to perform the export.
- Goggle APIs API key (here you can get one: https://code.google.com/apis/console/)
- Python 2.6 with components:
- - gdata-python-client (Google Python API) - http://code.google.com/p/gdata-python-client/
- - MySQLdb
- - PyQt4 (http://www.riverbankcomputing.co.uk/software/pyqt/download) --gui version only
+ - gdata-python-client (Google Python API) - http://code.google.com/p/gdata-python-client/
+ - MySQLdb
+ - PyQt4 (http://www.riverbankcomputing.co.uk/software/pyqt/download) --gui version only
Preparation
===========
Before running the script please be sure to:
- Prepare `google2piwik.conf` configuration file - sample in google2piwik.conf.sample
-- If you don't know your site table_id, type: python google2piwik.py -p
+- If you don't know your site table_id, type: python google2piwik.py -p
(remember to fill google login and pass configuration before)
- Check if configuration file is acceptable and all rights are present (./google2piwik.py -c)
- Create backup of your Piwik MySQL Database
@@ -77,9 +77,10 @@ Daniel Borzęcki
Maciej Zawadziński
Piotr Rzepecki
Maciej Sobczak
+Grzegorz Janik
Contact: office@clearcode.cc
License
=======
-http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
View
53 google2piwik.py
@@ -30,6 +30,7 @@
__VERBOSE__ = 0
SOURCE_APP_NAME = "Google2Piwik Exporter"
+CURRENT_VERSION = 1.9
def VERBOSE(message,level=0, new_line=True):
if __VERBOSE__ >= level: sys.stdout.write(message + '\n' if new_line else '')
@@ -112,15 +113,9 @@ def export_day(day, fetcher):
VERBOSE("VISIT: Initialize", 2)
simulator.initialize(fetcher, "ga:latitude,ga:longitude,ga:hour,ga:flashVersion,ga:javaEnabled,ga:language,ga:screenResolution", "ga:visits")
- VERBOSE("VISIT: Fetch 1", 2)
- simulator.update(fetcher, "ga:latitude,ga:longitude,ga:visitLength,ga:hour,ga:visitorType,ga:operatingSystem,ga:operatingSystemVersion","ga:visits")
- VERBOSE("VISIT: Fetch 2", 2)
- simulator.update(fetcher, "ga:latitude,ga:longitude,ga:hour,ga:browser,ga:browserVersion,ga:country,ga:continent","ga:visits")
- VERBOSE("VISIT: Fetch 3", 2)
- simulator.update(fetcher, "ga:longitude,ga:latitude,ga:hour,ga:browserVersion,ga:keyword,ga:source,ga:operatingSystemVersion","ga:visits")
-
- VERBOSE("VISIT: Fetch 4", 2)
- simulator.update(fetcher, "ga:longitude,ga:latitude,ga:hour,ga:visitCount,ga:daysSinceLastVisit","ga:visits")
+ for i, d in enumerate(dims.DVALS[CURRENT_VERSION]):
+ VERBOSE("VISIT: Fetch " + str(i), 2)
+ simulator.update(fetcher, d, "ga:visits")
"""
Getting landing and exit pages
@@ -139,7 +134,7 @@ def export_day(day, fetcher):
"""
VERBOSE("VISIT: Export vitis", 2)
for v in simulator.visits:
- v.idvisit = sql.insert_log_visit(v.visit_log)
+ v.idvisit = sql.insert_log_visit(v.visit_log, CURRENT_VERSION)
VERBOSE("VISIT: Completed", 2)
"""
@@ -237,7 +232,7 @@ def finalize(self, additional):
"""
self.visit_log.update(additional)
- stable = ["ga:screenResolution", "ga:language", "ga:visitLength", "total_actions", "ga:visitCount", "ga:daysSinceLastVisit"]
+ stable = ["ga:screenResolution", "ga:language", "ga:visitLength", "total_actions", "ga:visitCount", "ga:daysSinceLastVisit", "ga:city"]
for stable_dim in stable:
self.visit_log[dims.DMAP[stable_dim]] = self.google_data.get(stable_dim) or 0
@@ -246,22 +241,23 @@ def finalize(self, additional):
self.set_final("ga:javaEnabled", vals.java_present)
self.set_final("ga:hour", vals.visitor_localtime)
self.set_final("ga:browser", vals.browser_name)
- self.set_final("ga:continent", vals.continent_name)
self.set_final("ga:country", vals.country_name)
self.set_final("ga:keyword", vals.referer_keyword)
self.cut_final("ga:keyword", 255)
self.set_final("ga:source", vals.referer_url)
+ self.set_final("ga:continent", vals.continent_name)
+ self.set_final("ga:region", vals.region_name)
self.set_final_value("referer_type", vals.referer_type(self.google_data.get("ga:source"),
self.get_final_value("ga:keyword")))
self.set_final_value("referer_name", vals.referer_name(self.google_data.get("ga:source"),
self.get_final_value("referer_type")))
- self.cut_final("referer_name", 70)
+ self.cut_final("referer_name", 70)
self.set_final_value("ga:browserVersion", vals.browser_version(self.get_final_value("ga:browser"),
self.google_data.get("ga:browserVersion")))
- self.cut_final("ga:browserVersion", 20)
+ self.cut_final("ga:browserVersion", 20)
try:
landing_act_id = action_manager[self.google_data["ga:landingPagePath"]].id_action_url
@@ -520,8 +516,6 @@ def FeedToDict(self, take_dimension = True, take_metric = True):
return result
def PrintTableIDs(self):
- #account_query = gdata.analytics.client.AccountFeedQuery()
- #table_feed = self.client.GetAccountFeed(account_query)
account_query = gdata.analytics.client.ProfileQuery('~all', '~all',
{'key': config.GOOGLE_KEY})
table_feed = self.client.GetManagementFeed(account_query)
@@ -531,9 +525,10 @@ def PrintTableIDs(self):
entry.GetProperty('dxp:tableId').value)
def GetTableIDs(self):
- account_query = gdata.analytics.client.AccountFeedQuery()
- table_feed = self.client.GetAccountFeed(account_query)
- return [(entry.title.text,entry.table_id.text) for entry in table_feed.entry]
+ account_query = gdata.analytics.client.ProfileQuery('~all', '~all',
+ {'key': config.GOOGLE_KEY})
+ table_feed = self.client.GetManagementFeed(account_query)
+ return [(entry.GetProperty('ga:profileName').value, entry.GetProperty('dxp:tableId').value) for entry in table_feed.entry]
def EntryToDict(self, entry, take_dimension = True, take_metric = False):
result = {}
@@ -577,7 +572,7 @@ def EntryToDict(self, entry, take_dimension = True, take_metric = False):
fetcher = GoogleFeedFetcher("")
fetcher.PrintTableIDs()
exit()
-
+
if options.clear_archives:
try:
config.read_config(options.config_file)
@@ -600,19 +595,19 @@ def EntryToDict(self, entry, take_dimension = True, take_metric = False):
exit()
print
print "Checking Google Analytics"
-
+
#if not config.GOOGLE_USER.split("@")[1] == "gmail.com":
#print "Your e-mail address should be ending with @gmail.com"
#exit()
-
+
print "Attempting login:",
try:
fetcher = GoogleFeedFetcher( config.GOOGLE_TABLE_ID )
print "[OK]"
except:
print "[FAILED]"
-
-
+
+
print "Simple query on table:",
try:
fetcher.checkAccess()
@@ -621,8 +616,8 @@ def EntryToDict(self, entry, take_dimension = True, take_metric = False):
print "[OK]"
except:
print "[FAILED]"
-
- print
+
+ print
print "Checking MySQL Access"
print "Initialize database connection:",
try:
@@ -654,6 +649,10 @@ def EntryToDict(self, entry, take_dimension = True, take_metric = False):
start_date = read_date(options.start_date or config.CONFIG_START)
end_date = None if not (options.end_date or config.CONFIG_END) else read_date(options.end_date or config.CONFIG_END)
sql.initialize(config.MYSQL_CREDENTIALS)
+ CURRENT_VERSION = sql.get_version(config.MYSQL_CREDENTIALS["table_prefix"])
+ if (CURRENT_VERSION < 1.9):
+ CURRENT_VERSION = 1.8
+
if options.update_visit_actions:
sql.update_total_visit_actions()
exit()
@@ -663,4 +662,4 @@ def EntryToDict(self, entry, take_dimension = True, take_metric = False):
export_period(start_date, end_date)
sql.clear_archives()
- print "Please go to your Piwik installation folder and run misc/cron/archive.sh script."
+ print "Please go to your Piwik installation folder and run misc/cron/archive.sh script."
View
11,820 lib/regioncodes.pickle
11,820 additions, 0 deletions not shown
View
15 mappers/dimensions.py
@@ -13,6 +13,8 @@
"ga:browserVersion" : "config_browser_version",
"ga:continent" : "location_continent",
"ga:country" : "location_country",
+ "ga:city" : "location_city",
+ "ga:region" : "location_region",
"ga:flashVersion" : "config_flash",
"ga:javaEnabled" : "config_java",
"ga:screenResolution": "config_resolution",
@@ -36,3 +38,16 @@
"md5config" : "config_md5config",
"idcookie" : "visitor_idcookie"
}
+
+DVALS = {
+ 1.9: ["ga:latitude,ga:longitude,ga:visitLength,ga:hour,ga:visitorType,ga:operatingSystem,ga:operatingSystemVersion",
+ "ga:longitude,ga:latitude,ga:hour,ga:browserVersion,ga:keyword,ga:source,ga:operatingSystemVersion",
+ "ga:latitude,ga:longitude,ga:hour,ga:browser,ga:browserVersion,ga:country,ga:city",
+ "ga:longitude,ga:latitude,ga:hour,ga:visitCount,ga:daysSinceLastVisit,ga:region"
+ ],
+ 1.8: ["ga:latitude,ga:longitude,ga:visitLength,ga:hour,ga:visitorType,ga:operatingSystem,ga:operatingSystemVersion",
+ "ga:longitude,ga:latitude,ga:hour,ga:browserVersion,ga:keyword,ga:source,ga:operatingSystemVersion",
+ "ga:latitude,ga:longitude,ga:hour,ga:browser,ga:browserVersion,ga:country,ga:continent",
+ "ga:longitude,ga:latitude,ga:hour,ga:visitCount,ga:daysSinceLastVisit"
+ ]
+ }
View
7 mappers/values.py
@@ -8,6 +8,7 @@
import cPickle
country_codes = cPickle.load( open("lib/countrycodes.pickle") )
+region_codes = cPickle.load( open("lib/regioncodes.pickle") )
unknown = "UNK"
REFERER_TYPE_DIRECT_ENTRY = 1
@@ -128,4 +129,8 @@ def flash_present(value):
def java_present(value):
return int(value == "Yes")
-
+def region_name(value):
+ try:
+ return region_codes[value]
+ except KeyError:
+ return 'xx'
View
94 sql.py
@@ -18,34 +18,38 @@
T_LOGV = "log_visit"
T_SITE = "site"
-INSERT_LOG_VISIT_ACTION = """INSERT INTO {{LVA}} (idvisit, idvisitor, server_time, idsite, idaction_url,
+INSERT_LOG_VISIT_ACTION = """INSERT INTO {{LVA}} (idvisit, idvisitor, server_time, idsite, idaction_url,
idaction_url_ref, idaction_name, time_spent_ref_action, idaction_name_ref)
VALUES (%s, binary(unhex(substring(%s,1,16))), %s, %s, %s, %s, %s, %s, 0) """
-
-INSERT_LOG_ACTION = "INSERT INTO {{LA}} (name, hash, type) VALUES (%s, %s, %s) "
-INSERT_LOG_VISIT = u""" INSERT INTO {{LV}} (idsite, visitor_localtime, idvisitor, visitor_returning, config_id,
- visit_first_action_time, visit_last_action_time,
- visit_exit_idaction_url, visit_entry_idaction_url, visit_total_actions,
- visit_total_time, referer_type, referer_name, visit_goal_converted,
- referer_url, referer_keyword, config_os, config_browser_name,
- config_browser_version, config_resolution, config_pdf, config_flash,
- config_java, config_director, config_quicktime, config_realplayer,
- config_windowsmedia, config_gears, config_silverlight, config_cookie,
- location_ip, location_browser_lang, location_country, location_continent,
- visitor_count_visits, visitor_days_since_last, visitor_days_since_first,
- visit_exit_idaction_name, visit_entry_idaction_name)
-
- VALUES ( %(idsite)s, %(visitor_localtime)s, binary(unhex(substring(%(visitor_idcookie)s,1,16))),
- %(visitor_returning)s, binary(unhex(substring(%(config_md5config)s,1,16))),
- %(visit_first_action_time)s, %(visit_last_action_time)s,
- %(visit_exit_idaction_url)s, %(visit_entry_idaction_url)s, %(visit_total_actions)s,
- %(visit_total_time)s, %(referer_type)s, %(referer_name)s, 0,
- %(referer_url)s, %(referer_keyword)s, %(config_os)s, %(config_browser_name)s,
- %(config_browser_version)s, %(config_resolution)s, 0, %(config_flash)s,
- %(config_java)s, 0, 0, 0, 0, 0, 0, 0, 0,
- %(location_browser_lang)s, %(location_country)s, %(location_continent)s,
- %(visitor_count_visits)s, %(visitor_days_since_last)s, 0, 0, 0) """
+INSERT_LOG_ACTION = "INSERT INTO {{LA}} (name, hash, type) VALUES (%s, %s, %s) "
+LOGV_TEMPLATE = u""" INSERT INTO {{LV}} (idsite, visitor_localtime, idvisitor, visitor_returning, config_id,
+ visit_first_action_time, visit_last_action_time,
+ visit_exit_idaction_url, visit_entry_idaction_url, visit_total_actions,
+ visit_total_time, referer_type, referer_name, visit_goal_converted,
+ referer_url, referer_keyword, config_os, config_browser_name,
+ config_browser_version, config_resolution, config_pdf, config_flash,
+ config_java, config_director, config_quicktime, config_realplayer,
+ config_windowsmedia, config_gears, config_silverlight, config_cookie,
+ location_ip, location_browser_lang, location_country{0},
+ visitor_count_visits, visitor_days_since_last, visitor_days_since_first,
+ visit_exit_idaction_name, visit_entry_idaction_name{1})
+
+ VALUES ( %(idsite)s, %(visitor_localtime)s, binary(unhex(substring(%(visitor_idcookie)s,1,16))),
+ %(visitor_returning)s, binary(unhex(substring(%(config_md5config)s,1,16))),
+ %(visit_first_action_time)s, %(visit_last_action_time)s,
+ %(visit_exit_idaction_url)s, %(visit_entry_idaction_url)s, %(visit_total_actions)s,
+ %(visit_total_time)s, %(referer_type)s, %(referer_name)s, 0,
+ %(referer_url)s, %(referer_keyword)s, %(config_os)s, %(config_browser_name)s,
+ %(config_browser_version)s, %(config_resolution)s, 0, %(config_flash)s,
+ %(config_java)s, 0, 0, 0, 0, 0, 0, 0, 0,
+ %(location_browser_lang)s, %(location_country)s{2},
+ %(visitor_count_visits)s, %(visitor_days_since_last)s, 0, 0, 0{3}) """
+
+INSERT_LOG_VISIT = {
+ 1.9: (", location_city", ", location_region", ", %(location_city)s", ", %(location_region)s"),
+ 1.8: (", location_continent", "", ", %(location_continent)s", "")
+}
SELECT_NB_VISITS = "SELECT count(*) FROM {{LV}} WHERE visitor_localtime = %s and idsite = %s"
@@ -54,17 +58,21 @@ def initialize(mysql_data):
global db, cursor
global INSERT_LOG_VISIT_ACTION, INSERT_LOG_ACTION, INSERT_LOG_VISIT
global SELECT_NB_VISITS
+ global LOGV_TEMPLATE
prefix = mysql_data["table_prefix"]
T_LOGVA = "%s_%s" % (prefix, T_LOGVA) if prefix else T_LOGVA
T_LOGA = "%s_%s" % (prefix, T_LOGA) if prefix else T_LOGA
T_LOGV = "%s_%s" % (prefix, T_LOGV) if prefix else T_LOGV
T_SITE = "%s_%s" % (prefix, T_SITE) if prefix else T_SITE
-
+
INSERT_LOG_VISIT_ACTION = INSERT_LOG_VISIT_ACTION.replace("{{LVA}}", T_LOGVA)
INSERT_LOG_ACTION = INSERT_LOG_ACTION.replace("{{LA}}", T_LOGA)
- INSERT_LOG_VISIT = INSERT_LOG_VISIT.replace("{{LV}}", T_LOGV)
SELECT_NB_VISITS = SELECT_NB_VISITS.replace("{{LV}}", T_LOGV)
-
+
+ LOGV_TEMPLATE = LOGV_TEMPLATE.replace("{{LV}}", T_LOGV)
+ for k, v in INSERT_LOG_VISIT.iteritems():
+ INSERT_LOG_VISIT[k] = LOGV_TEMPLATE.format(*INSERT_LOG_VISIT[k])
+
db = init_db(mysql_data)
db.set_character_set('utf8')
cursor = db.cursor()
@@ -72,10 +80,10 @@ def initialize(mysql_data):
def insert_log_action(values):
cursor.execute(INSERT_LOG_ACTION, values)
return cursor.lastrowid
-
-def insert_log_visit(values):
+
+def insert_log_visit(values, version):
try:
- cursor.execute(INSERT_LOG_VISIT, values)
+ cursor.execute(INSERT_LOG_VISIT[version], values)
except:
pass
return cursor.lastrowid
@@ -95,17 +103,24 @@ def init_db(mysql_data):
exit()
def test_db(mysql_data):
- global db, cursor
- db = MySQLdb.connect(mysql_data["host"], mysql_data["user"], mysql_data["passwd"],
+ global db, cursor
+ db = MySQLdb.connect(mysql_data["host"], mysql_data["user"], mysql_data["passwd"],
mysql_data["db"], int(mysql_data["port"]))
- db.set_character_set('utf8')
- cursor = db.cursor()
-
+ db.set_character_set('utf8')
+ cursor = db.cursor()
+
def get_sites(prefix):
select_site_sql = "SELECT idsite, name, main_url from {SITE_TABLE}".format(SITE_TABLE = prefix+"_"+T_SITE)
cursor.execute(select_site_sql)
return [{"id" : id, "name" : name, "url" : url} for (id, name, url) in cursor.fetchall()]
-
+
+def get_version(prefix):
+ t_option = "%s_option" % (prefix) if prefix else "option"
+ select_version_sql = "SELECT option_value FROM {table} WHERE option_name = 'version_core'".format(table = t_option)
+ cursor.execute(select_version_sql)
+ version = float(cursor.fetchone()[0][:3])
+ return version
+
def check_tables(table_prefix):
global cursor
failed = []
@@ -128,7 +143,7 @@ def update_site_ts_created(site_id, date):
select_site_sql = "SELECT ts_created from {SITE_TABLE} WHERE idsite = %s".format(SITE_TABLE = T_SITE)
cursor.execute(select_site_sql, site_id)
ts_created = cursor.fetchone()[0]
-
+
if ts_created > current_start:
update_site_sql = "UPDATE {SITE_TABLE} SET ts_created = %s WHERE idsite = %s".format(SITE_TABLE = T_SITE)
cursor.execute(update_site_sql, (current_start, site_id))
@@ -136,7 +151,7 @@ def update_site_ts_created(site_id, date):
def nb_visits_day(date, site_id):
cursor.execute(SELECT_NB_VISITS,(date, site_id))
return cursor.fetchone()[0]
-
+
def update_visit_actions(start_date, end_date):
raw_sql = """UPDATE {LV} AS lv
LEFT JOIN (
@@ -177,4 +192,3 @@ def clear_archives():
if to_drop:
raw_sql = 'DROP TABLE ' + (', ').join(to_drop)
cursor.execute(raw_sql)
-
Please sign in to comment.
Something went wrong with that request. Please try again.