merge with latest master

integration_tests/test_query.py

@@ -1,7 +1,7 @@
 from splice.environment import Environment
 
-# env = Environment.instance("integration_tests.tim_settings.DefaultConfig")
-env = Environment.instance()
+env = Environment.instance("integration_tests.prod_settings.DefaultConfig")
+# env = Environment.instance()
 
 from splice.queries import tile_stats_weekly, slot_stats_weekly, tile_stats_monthly, slot_stats_monthly, \
     tile_summary_weekly, slot_summary_weekly, tile_summary_monthly, slot_summary_monthly, \
@@ -12,33 +12,33 @@
     conn = env.db.engine.connect()
     print "\ntile_summary_weekly"
     key, rval = tile_summary_weekly(conn, '2014-05-15')
-    for year, week, tile_id, imps, clicks, pinned, blocked, spon, spon_link, newtabs in rval:
-        print year, week, tile_id, imps, clicks, pinned, blocked, spon, spon_link, newtabs
+    for x in rval:
+        print x
 
     print "\ntile_summary_daily"
     _, rval = tile_summary_daily(conn, '2014-05-15')
-    for year, week, tile_id, imps, clicks, pinned, blocked, spon, spon_link, newtabs in rval:
-        print year, week, tile_id, imps, clicks, pinned, blocked, spon, spon_link, newtabs
+    for year, week, tile_id, title, imps, clicks, pinned, blocked, spon, spon_link, newtabs in rval:
+        print year, week, tile_id, title, imps, clicks, pinned, blocked, spon, spon_link, newtabs
 
     print "\ntile_stats_weekly - tile_id = 2"
     _, rval = tile_stats_weekly(conn, '2014-05-15', '2')
-    for year, week, tile_id, country, locale, imps, clicks, pinned, blocked, spon, spon_link, newtabs in rval:
-        print year, week, tile_id, country, locale, imps, clicks, pinned, blocked, spon, spon_link, newtabs
+    for year, week, tile_id, title, country, locale, imps, clicks, pinned, blocked, spon, spon_link, newtabs in rval:
+        print year, week, tile_id, title, country, locale, imps, clicks, pinned, blocked, spon, spon_link, newtabs
 
     print "\ntile_stats_weekly - tile_id = 2, country_code = US"
     _, rval = tile_stats_weekly(conn, '2014-05-15', '2', 'US')
-    for year, week, tile_id, country, locale, imps, clicks, pinned, blocked, spon, spon_link, newtabs in rval:
-        print year, week, tile_id, country, locale, imps, clicks, pinned, blocked, spon, spon_link, newtabs
+    for year, week, tile_id, title, country, locale, imps, clicks, pinned, blocked, spon, spon_link, newtabs in rval:
+        print year, week, tile_id, title, country, locale, imps, clicks, pinned, blocked, spon, spon_link, newtabs
 
     print "\ntile_stats_daily - tile_id = 2"
     _, rval = tile_stats_daily(conn, '2014-05-15', '2')
-    for year, week, tile_id, country, locale, imps, clicks, pinned, blocked, spon, spon_link, newtabs in rval:
-        print year, week, tile_id, country, locale, imps, clicks, pinned, blocked, spon, spon_link, newtabs
+    for year, week, tile_id, title, country, locale, imps, clicks, pinned, blocked, spon, spon_link, newtabs in rval:
+        print year, week, tile_id, title, country, locale, imps, clicks, pinned, blocked, spon, spon_link, newtabs
 
     print "\ntile_stats_daily - tile_id = 2, country_code = US"
     _, rval = tile_stats_daily(conn, '2014-05-15', '2', 'US')
-    for year, week, tile_id, country, locale, imps, clicks, pinned, blocked, spon, spon_link, newtabs in rval:
-        print year, week, tile_id, country, locale, imps, clicks, pinned, blocked, spon, spon_link, newtabs
+    for year, week, tile_id, title, country, locale, imps, clicks, pinned, blocked, spon, spon_link, newtabs in rval:
+        print year, week, tile_id, title, country, locale, imps, clicks, pinned, blocked, spon, spon_link, newtabs
 
     print "\nslot_summary_weekly"
     _, rval = slot_summary_weekly(conn, '2014-05-15')
@@ -57,18 +57,18 @@
 
     print "\ntile_summary_monthly"
     _, rval = tile_summary_monthly(conn, '2014-05-15')
-    for year, week, tile_id, imps, clicks, pinned, blocked, spon, spon_link, newtabs in rval:
-        print year, week, tile_id, imps, clicks, pinned, blocked, spon, spon_link, newtabs
+    for year, week, tile_id, title, imps, clicks, pinned, blocked, spon, spon_link, newtabs in rval:
+        print year, week, tile_id, title, imps, clicks, pinned, blocked, spon, spon_link, newtabs
 
     print "\ntile_stats_monthly - tile_id = 2"
     _, rval = tile_stats_monthly(conn, '2014-05-15', '2')
-    for year, week, tile_id, country, locale, imps, clicks, pinned, blocked, spon, spon_link, newtabs in rval:
-        print year, week, tile_id, country, locale, imps, clicks, pinned, blocked, spon, spon_link, newtabs
+    for year, week, tile_id, title, country, locale, imps, clicks, pinned, blocked, spon, spon_link, newtabs in rval:
+        print year, week, tile_id, title, country, locale, imps, clicks, pinned, blocked, spon, spon_link, newtabs
 
     print "\ntile_stats_monthly - tile_id = 2, country_code = US"
     _, rval = tile_stats_monthly(conn, '2014-05-15', '2', 'US')
-    for year, week, tile_id, country, locale, imps, clicks, pinned, blocked, spon, spon_link, newtabs in rval:
-        print year, week, tile_id, country, locale, imps, clicks, pinned, blocked, spon, spon_link, newtabs
+    for year, week, tile_id, title, country, locale, imps, clicks, pinned, blocked, spon, spon_link, newtabs in rval:
+        print year, week, tile_id, title, country, locale, imps, clicks, pinned, blocked, spon, spon_link, newtabs
 
     print "\nslot_summary_monthly"
     _, rval = slot_summary_monthly(conn, '2014-05-15')

setup.py

@@ -2,7 +2,7 @@
 
 setup(
     name="splice",
-    version="1.0.8",
+    version="1.0.9",
     description="Link authoring tool for " +
                 "Firefox Directory and Enhanced Tiles",
     author="Mozilla",

splice/models.py

@@ -62,7 +62,6 @@ class UniqueCountsDaily(db.Model):
     db.Column('blocked', db.Integer, nullable=False, server_default="0"),
     db.Column('sponsored_link', db.Integer, nullable=False, server_default="0"),
     db.Column('sponsored', db.Integer, nullable=False, server_default="0"),
-    db.Column('newtabs', db.Integer, nullable=False, server_default="0"),
     db.Column('position', db.Integer, nullable=False, server_default="0"),
     db.Column('enhanced', db.Boolean, nullable=False, server_default="false"),
     db.Column('locale', db.String(14), nullable=False),
@@ -71,7 +70,8 @@ class UniqueCountsDaily(db.Model):
     db.Column('browser', db.String(64), nullable=False),
     db.Column('version', db.String(64), nullable=False),
     db.Column('device', db.String(64), nullable=False),
-    db.Column('year', db.Integer, nullable=False),
     db.Column('month', db.Integer, nullable=False),
     db.Column('week', db.Integer, nullable=False),
+    db.Column('year', db.Integer, nullable=False),
+    db.Column('newtabs', db.Integer, nullable=False, server_default="0"),
 )

splice/queries.py

@@ -29,35 +29,76 @@ def tile_exists(target_url, bg_color, title, type, image_uri, enhanced_image_uri
     return results
 
 
-def _stats_query(connection, start_date, date_window, group_column_name, group_value, country_code):
+def _slot_query(connection, start_date, date_window, position, country_code):
     dt = datetime.strptime(start_date, "%Y-%m-%d")
     year = dt.year
     if date_window == 'month':
         window_param = dt.month
     elif date_window == 'date':
-        window_param = dt
+        window_param = dt.date()
     else:
         window_param = dt.isocalendar()[1]
 
     imps = impression_stats_daily
     window_func_table = imps.c.get(date_window)
-    group_column = imps.c.get(group_column_name)
 
     # the where clause is an ANDed list of country, monthly|weekly, and year conditions
-    where_elements = [imps.c.year >= year, window_func_table >= window_param]
+    where_elements = [imps.c.year >= year, window_func_table >= window_param, imps.c.position == position]
     if country_code is not None:
         where_elements.append(imps.c.country_code == country_code)
 
-    if group_value is not None:
-        where_elements.append(group_column == group_value)
+    where_clause = and_(*where_elements)
+
+    stmt = select(
+        [
+            imps.c.year,
+            window_func_table,
+            imps.c.position,
+            imps.c.country_code,
+            imps.c.locale,
+            func.sum(imps.c.impressions),
+            func.sum(imps.c.clicks),
+            func.sum(imps.c.pinned),
+            func.sum(imps.c.blocked),
+            func.sum(imps.c.sponsored),
+            func.sum(imps.c.sponsored_link),
+            func.sum(imps.c.newtabs)
+        ]) \
+        .where(where_clause) \
+        .group_by(imps.c.year, window_func_table, imps.c.position, imps.c.country_code, imps.c.locale) \
+        .order_by(imps.c.year, window_func_table, imps.c.position, imps.c.country_code, imps.c.locale)
+    return ('year', date_window, 'position', 'country_code', 'locale',
+            'impressions', 'clicks', 'pinned', 'blocked', 'sponsored', 'sponsored_link', 'newtabs'), \
+        connection.execute(stmt)
+
+
+def _tile_query(connection, start_date, date_window, tile_id, country_code):
+    dt = datetime.strptime(start_date, "%Y-%m-%d")
+    year = dt.year
+    if date_window == 'month':
+        window_param = dt.month
+    elif date_window == 'date':
+        window_param = dt.date()
+    else:
+        window_param = dt.isocalendar()[1]
+
+    imps = impression_stats_daily
+    window_func_table = imps.c.get(date_window)
+
+    # the where clause is an ANDed list of country, monthly|weekly, and year conditions
+    where_elements = [imps.c.year >= year, window_func_table >= window_param,
+                      imps.c.tile_id == tile_id, imps.c.tile_id == Tile.id]
+    if country_code is not None:
+        where_elements.append(imps.c.country_code == country_code)
 
     where_clause = and_(*where_elements)
 
     stmt = select(
         [
             imps.c.year,
             window_func_table,
-            group_column,
+            imps.c.tile_id,
+            Tile.title,
             imps.c.country_code,
             imps.c.locale,
             func.sum(imps.c.impressions),
@@ -69,26 +110,68 @@ def _stats_query(connection, start_date, date_window, group_column_name, group_v
             func.sum(imps.c.newtabs)
         ]) \
         .where(where_clause) \
-        .group_by(imps.c.year, window_func_table, group_column, imps.c.country_code, imps.c.locale) \
-        .order_by(imps.c.year, window_func_table, group_column, imps.c.country_code, imps.c.locale)
-    return ('year', date_window, group_column_name, 'country_code', 'locale',
+        .group_by(imps.c.year, window_func_table, imps.c.tile_id, Tile.title, imps.c.country_code, imps.c.locale) \
+        .order_by(imps.c.year, window_func_table, imps.c.tile_id, imps.c.country_code, imps.c.locale)
+    return ('year', date_window, 'tile_id', 'tile_title', 'country_code', 'locale',
+            'impressions', 'clicks', 'pinned', 'blocked', 'sponsored', 'sponsored_link', 'newtabs'), \
+        connection.execute(stmt)
+
+
+def _tile_summary_query(connection, start_date, date_window, country_code):
+    dt = datetime.strptime(start_date, "%Y-%m-%d")
+    year = dt.year
+    if date_window == 'month':
+        window_param = dt.month
+    elif date_window == 'date':
+        window_param = dt.date()
+    else:
+        window_param = dt.isocalendar()[1]
+
+    imps = impression_stats_daily
+    window_func_table = imps.c.get(date_window)
+
+    # the where clause is an ANDed list of country, monthly|weekly, and year conditions
+    where_elements = [imps.c.year >= year, window_func_table >= window_param, imps.c.tile_id == Tile.id]
+    if country_code is not None:
+        where_elements.append(imps.c.country_code == country_code)
+
+    where_clause = and_(*where_elements)
+
+    stmt = select(
+        [
+            imps.c.year,
+            window_func_table,
+            imps.c.tile_id,
+            Tile.title,
+            func.sum(imps.c.impressions),
+            func.sum(imps.c.clicks),
+            func.sum(imps.c.pinned),
+            func.sum(imps.c.blocked),
+            func.sum(imps.c.sponsored),
+            func.sum(imps.c.sponsored_link),
+            func.sum(imps.c.newtabs)
+        ]) \
+        .where(where_clause) \
+        .group_by(imps.c.year, window_func_table, imps.c.tile_id, Tile.title) \
+        .order_by(imps.c.year, window_func_table, imps.c.tile_id)
+
+    return ('year', date_window, 'tile_id', 'tile_title',
             'impressions', 'clicks', 'pinned', 'blocked', 'sponsored', 'sponsored_link', 'newtabs'), \
         connection.execute(stmt)
 
 
-def _summary_query(connection, start_date, date_window, group_column_name, country_code):
+def _slot_summary_query(connection, start_date, date_window, country_code):
     dt = datetime.strptime(start_date, "%Y-%m-%d")
     year = dt.year
     if date_window == 'month':
         window_param = dt.month
     elif date_window == 'date':
-        window_param = dt
+        window_param = dt.date()
     else:
         window_param = dt.isocalendar()[1]
 
     imps = impression_stats_daily
     window_func_table = imps.c.get(date_window)
-    group_column = imps.c.get(group_column_name)
 
     # the where clause is an ANDed list of country, monthly|weekly, and year conditions
     where_elements = [imps.c.year >= year, window_func_table >= window_param]
@@ -101,7 +184,7 @@ def _summary_query(connection, start_date, date_window, group_column_name, count
         [
             imps.c.year,
             window_func_table,
-            group_column,
+            imps.c.position,
             func.sum(imps.c.impressions),
             func.sum(imps.c.clicks),
             func.sum(imps.c.pinned),
@@ -111,51 +194,51 @@ def _summary_query(connection, start_date, date_window, group_column_name, count
             func.sum(imps.c.newtabs)
         ]) \
         .where(where_clause) \
-        .group_by(imps.c.year, window_func_table, group_column) \
-        .order_by(imps.c.year, window_func_table, group_column)
-    return ('year', date_window, group_column_name,
+        .group_by(imps.c.year, window_func_table, imps.c.position) \
+        .order_by(imps.c.year, window_func_table, imps.c.position)
+    return ('year', date_window, 'position',
             'impressions', 'clicks', 'pinned', 'blocked', 'sponsored', 'sponsored_link', 'newtabs'), \
         connection.execute(stmt)
 
 
 def tile_stats_weekly(connection, start_date, tile_id=None, country_code=None):
-    return _stats_query(connection, start_date, 'week', 'tile_id', tile_id, country_code)
+    return _tile_query(connection, start_date, 'week', tile_id, country_code)
 
 
 def tile_stats_monthly(connection, start_date, tile_id=None, country_code=None):
-    return _stats_query(connection, start_date, 'month', 'tile_id', tile_id, country_code)
+    return _tile_query(connection, start_date, 'month', tile_id, country_code)
 
 
 def tile_stats_daily(connection, start_date, tile_id=None, country_code=None):
-    return _stats_query(connection, start_date, 'date', 'tile_id', tile_id, country_code)
+    return _tile_query(connection, start_date, 'date', tile_id, country_code)
 
 
 def tile_summary_weekly(connection, start_date, country_code=None):
-    return _summary_query(connection, start_date, 'week', 'tile_id', country_code)
+    return _tile_summary_query(connection, start_date, 'week', country_code)
 
 
 def tile_summary_monthly(connection, start_date, country_code=None):
-    return _summary_query(connection, start_date, 'month', 'tile_id', country_code)
+    return _tile_summary_query(connection, start_date, 'month', country_code)
 
 
 def tile_summary_daily(connection, start_date, country_code=None):
-    return _summary_query(connection, start_date, 'date', 'tile_id', country_code)
+    return _tile_summary_query(connection, start_date, 'date', country_code)
 
 
-def slot_stats_weekly(connection, start_date, slot_id=None, country_code=None):
-    return _stats_query(connection, start_date, 'week', 'position', slot_id, country_code)
+def slot_stats_weekly(connection, start_date, position=None, country_code=None):
+    return _slot_query(connection, start_date, 'week', position, country_code)
 
 
-def slot_stats_monthly(connection, start_date, slot_id=None, country_code=None):
-    return _stats_query(connection, start_date, 'month', 'position', slot_id, country_code)
+def slot_stats_monthly(connection, start_date, position=None, country_code=None):
+    return _slot_query(connection, start_date, 'month', position, country_code)
 
 
 def slot_summary_weekly(connection, start_date, country_code=None):
-    return _summary_query(connection, start_date, 'week', 'position', country_code)
+    return _slot_summary_query(connection, start_date, 'week', country_code)
 
 
 def slot_summary_monthly(connection, start_date, country_code=None):
-    return _summary_query(connection, start_date, 'month', 'position', country_code)
+    return _slot_summary_query(connection, start_date, 'month', country_code)
 
 
 def insert_tile(target_url, bg_color, title, type, image_uri, enhanced_image_uri, locale, *args, **kwargs):